Add --evict-older-than (#605)

The argument adds another mechanism to control contents of cache directory. And
is based on the LRU tracking behaviour.

As of now there is no way for ccache to eliminate files which were
are no longer needed. As a result these files stay and are kept around until
either max_files/max_size is reached.

If a particular project is being built regularly but for some reason is allowed
to grow in size, then under such circumstances using the LRU mechanism to control
cache size, lends as perfect solution.

The argument takes a parameter N and performs a cleanup.
While performing cleanup the oldest file in ccache can
only be N seconds old. However this cleanup will not take max_files and
max_old into consideration

diff --git a/doc/MANUAL.adoc b/doc/MANUAL.adoc
index 7cd035c713a16940d558e734bd5604cb9e4d0727..c4bbad56f577952014ee6c96f52c8c795e89cc56 100644 (file)
--- a/doc/MANUAL.adoc
+++ b/doc/MANUAL.adoc
@@ -94,6 +94,16 @@ Common options
     Clear the entire cache, removing all cached files, but keeping the
     configuration file.
 
+*`--evict-older-than`* _NUM_::
+
+    This is another cleanup mechanism (independent of max_files and max_size).
+    It instructs ccache to perform cleanup based on age of the stored files.
+    The value _NUM_ is duration and means any cache entry older than it from CURRENT_TIME
+    should be deleted. _NUM_ should be an unsigned number with a suffix:
+    d(ays)/s(econds).
+    This cleanup can be performed from the cli with this option only. During the
+    cleanup max_files/max_size will not be taken into consideration.
+
 *`-h`*, *`--help`*::
 
     Print an options summary page.

diff --git a/src/Util.cpp b/src/Util.cpp
index cef42b20c759209caaf1d622627806695d7f547b..b20c549a1c42473d079bd521ab9dfed54506addd 100644 (file)
--- a/src/Util.cpp
+++ b/src/Util.cpp
@@ -976,4 +976,37 @@ write_file(const std::string& path,
   file << data;
 }
 
+unsigned
+parse_duration_with_suffix_to_seconds(const std::string& value)
+{
+  size_t end;
+  long result;
+  bool failed = false;
+
+  try {
+    result = std::stol(value, &end, 10);
+  } catch (std::exception&) {
+    failed = true;
+  }
+
+  if (failed || result < 0) {
+    throw Error(fmt::format("invalid unsigned integer: \"{}\"", value));
+  }
+
+  if (end + 1 != value.size()) {
+    throw Error(
+      fmt::format("Invalid suffix, Supported: d(ay)/s(econd): \"{}\"", value));
+  }
+
+  switch (value[end]) {
+  case 'd':
+    result *= 24 * 3600;
+  case 's':
+    break;
+  default:
+    throw Error(
+      fmt::format("Invalid suffix, Supported: d(ay)/s(econd): \"{}\"", value));
+  }
+  return result;
+}
 } // namespace Util

diff --git a/src/Util.hpp b/src/Util.hpp
index 4492fc846d11a84093b517af4753c9a6683d4bd2..10f5fdb8a0d8a1bc29065c4de90104bceecf1245 100644 (file)
--- a/src/Util.hpp
+++ b/src/Util.hpp
@@ -347,4 +347,11 @@ void write_file(const std::string& path,
                 const std::string& data,
                 std::ios_base::openmode open_mode = std::ios::binary);
 
+// Parse the given string into an unsigned integer. Then based on suffix
+// provided convert the number to seconds possible suffixes = d(ays)/s(econds)
+//
+// Throws `Error` for any other suffix
+// Throws `Error` if parse value is <0
+unsigned parse_duration_with_suffix_to_seconds(const std::string& value);
+
 } // namespace Util

diff --git a/src/ccache.cpp b/src/ccache.cpp
index 057f50e9d9507b11f74e1674b276c0f415d057b3..c12c6a854b562760b5c260d18486cad140a1f102 100644 (file)
--- a/src/ccache.cpp
+++ b/src/ccache.cpp
@@ -89,6 +89,10 @@ Common options:
                               (normally not needed as this is done
                               automatically)
     -C, --clear               clear the cache completely (except configuration)
+        --evict-older-than N  delete files older than N (days/seconds) (this will not
+                              take max_files, max_size into consideration).
+                              N should be an unsigned number with a suffix:
+                              d(ays)/s(econds).
     -F, --max-files NUM       set maximum number of files in cache to NUM (use 0
                               for no limit)
     -M, --max-size SIZE       set maximum size of cache to SIZE (use 0 for no
@@ -2200,6 +2204,7 @@ handle_main_options(int argc, const char* const* argv)
     DUMP_MANIFEST,
     DUMP_RESULT,
     EXTRACT_RESULT,
+    EVICT_OLDER_THAN,
     HASH_FILE,
     PRINT_STATS,
   };
@@ -2209,6 +2214,7 @@ handle_main_options(int argc, const char* const* argv)
     {"dump-manifest", required_argument, nullptr, DUMP_MANIFEST},
     {"dump-result", required_argument, nullptr, DUMP_RESULT},
     {"extract-result", required_argument, nullptr, EXTRACT_RESULT},
+    {"evict-older-than", required_argument, nullptr, EVICT_OLDER_THAN},
     {"get-config", required_argument, nullptr, 'k'},
     {"hash-file", required_argument, nullptr, HASH_FILE},
     {"help", no_argument, nullptr, 'h'},
@@ -2258,6 +2264,17 @@ handle_main_options(int argc, const char* const* argv)
       return error ? EXIT_FAILURE : EXIT_SUCCESS;
     }
 
+    case EVICT_OLDER_THAN: {
+      unsigned seconds = Util::parse_duration_with_suffix_to_seconds(optarg);
+      ProgressBar progress_bar("Clearing ...");
+      clean_old(
+        ctx, [&](double progress) { progress_bar.update(progress); }, seconds);
+      if (isatty(STDOUT_FILENO)) {
+        printf("\n");
+      }
+      break;
+    }
+
     case HASH_FILE: {
       Hash hash;
       if (str_eq(optarg, "-")) {

diff --git a/src/cleanup.cpp b/src/cleanup.cpp
index 8423301d563258b6dbe85553ab12d8a93e40beb0..1e41cde1156f70f209d63cc2fc62dc666d922acf 100644 (file)
--- a/src/cleanup.cpp
+++ b/src/cleanup.cpp
@@ -57,6 +57,7 @@ void
 clean_up_dir(const std::string& subdir,
              uint64_t max_size,
              uint32_t max_files,
+             time_t max_age,
              const Util::ProgressReceiver& progress_receiver)
 {
   cc_log("Cleaning up cache directory %s", subdir.c_str());
@@ -111,7 +112,8 @@ clean_up_dir(const std::string& subdir,
     }
 
     if ((max_size == 0 || cache_size <= max_size)
-        && (max_files == 0 || files_in_cache <= max_files)) {
+        && (max_files == 0 || files_in_cache <= max_files)
+        && (max_age == 0 || file->lstat().mtime() > (current_time - max_age))) {
       break;
     }
 
@@ -163,6 +165,7 @@ clean_up_all(const Config& config,
       clean_up_dir(subdir,
                    config.max_size() / 16,
                    config.max_files() / 16,
+                   0,
                    sub_progress_receiver);
     },
     progress_receiver);
@@ -202,3 +205,17 @@ wipe_all(const Context& ctx, const Util::ProgressReceiver& progress_receiver)
   ctx.inode_cache.drop();
 #endif
 }
+
+void
+clean_old(const Context& ctx,
+          const Util::ProgressReceiver& progress_receiver,
+          time_t max_age)
+{
+  Util::for_each_level_1_subdir(
+    ctx.config.cache_dir(),
+    [&](const std::string& subdir,
+        const Util::ProgressReceiver& sub_progress_receiver) {
+      clean_up_dir(subdir, 0, 0, max_age, sub_progress_receiver);
+    },
+    progress_receiver);
+}

diff --git a/src/cleanup.hpp b/src/cleanup.hpp
index 1e34e66d4a13fbf907831941aa89b0c3d14f42d2..5148bf91ef3d506d06d1bed0853bfa638216db58 100644 (file)
--- a/src/cleanup.hpp
+++ b/src/cleanup.hpp
@@ -30,6 +30,7 @@ class Context;
 void clean_up_dir(const std::string& subdir,
                   uint64_t max_size,
                   uint32_t max_files,
+                  time_t max_age,
                   const Util::ProgressReceiver& progress_receiver);
 
 void clean_up_all(const Config& config,
@@ -37,3 +38,7 @@ void clean_up_all(const Config& config,
 
 void wipe_all(const Context& ctx,
               const Util::ProgressReceiver& progress_receiver);
+
+void clean_old(const Context& ctx,
+               const Util::ProgressReceiver& progress_receiver,
+               time_t max_age);

diff --git a/src/stats.cpp b/src/stats.cpp
index ec397957215189838edbc16e2ea412b02e3f03a9..4fe1966f4f47732f19f38d120980598f2d5da588 100644 (file)
--- a/src/stats.cpp
+++ b/src/stats.cpp
@@ -372,7 +372,9 @@ stats_flush_to_file(const Config& config,
     double factor = config.limit_multiple() / 16;
     uint64_t max_size = round(config.max_size() * factor);
     uint32_t max_files = round(config.max_files() * factor);
-    clean_up_dir(subdir, max_size, max_files, [](double /*progress*/) {});
+    uint32_t max_age = 0;
+    clean_up_dir(
+      subdir, max_size, max_files, max_age, [](double /*progress*/) {});
   }
 }
 

diff --git a/test/suites/cleanup.bash b/test/suites/cleanup.bash
index af2f4f7afc20c98dae411bc3901076dd2dcbad10..a85088c79d13185474a70f4d8486c5bff5f0ac64 100644 (file)
--- a/test/suites/cleanup.bash
+++ b/test/suites/cleanup.bash
@@ -179,4 +179,22 @@ SUITE_cleanup() {
     $CCACHE -c >/dev/null
     expect_file_count 1 '.nfs*' $CCACHE_DIR
     expect_stat 'files in cache' 10
+    # -------------------------------------------------------------------------
+    TEST "cleanup of old files by age"
+
+    prepare_cleanup_test_dir $CCACHE_DIR/a
+    touch $CCACHE_DIR/a/now.result
+    $CCACHE -F 0 -M 0 >/dev/null
+
+    $CCACHE --evict-older-than 1d >/dev/null
+    expect_file_count 1 '*.result' $CCACHE_DIR
+    expect_stat 'files in cache' 1
+
+    $CCACHE --evict-older-than 1d  >/dev/null
+    expect_file_count 1 '*.result' $CCACHE_DIR
+    expect_stat 'files in cache' 1
+
+    backdate $CCACHE_DIR/a/now.result
+    $CCACHE --evict-older-than 10s  >/dev/null
+    expect_stat 'files in cache' 0
 }

diff --git a/unittest/test_Util.cpp b/unittest/test_Util.cpp
index ce4638af0384340aebbcff5b64a8792a3d6ec62c..71586e07fc4faaa0d81ac87c8f1aa874e5c439ab 100644 (file)
--- a/unittest/test_Util.cpp
+++ b/unittest/test_Util.cpp
@@ -766,4 +766,18 @@ TEST_CASE("Util::wipe_path")
   }
 }
 
+TEST_CASE("Util::parse_duration_with_suffix_to_seconds")
+{
+  CHECK(Util::parse_duration_with_suffix_to_seconds("0s") == 0);
+  CHECK(Util::parse_duration_with_suffix_to_seconds("2s") == 2);
+  CHECK(Util::parse_duration_with_suffix_to_seconds("1d") == 3600 * 24);
+  CHECK(Util::parse_duration_with_suffix_to_seconds("2d") == 2 * 3600 * 24);
+  CHECK_THROWS_WITH(Util::parse_duration_with_suffix_to_seconds("-2"),
+                    "invalid unsigned integer: \"-2\"");
+  CHECK_THROWS_WITH(Util::parse_duration_with_suffix_to_seconds("2x"),
+                    "Invalid suffix, Supported: d(ay)/s(econd): \"2x\"");
+  CHECK_THROWS_WITH(Util::parse_duration_with_suffix_to_seconds("2"),
+                    "Invalid suffix, Supported: d(ay)/s(econd): \"2\"");
+}
+
 TEST_SUITE_END();