From: Sumit Jamgade <25170818+sjamgade@users.noreply.github.com>
Date: Sun, 26 Jul 2020 18:14:19 +0000 (+0200)
Subject: Add --evict-older-than (#605)
X-Git-Tag: v4.0~292
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=87ae5e39e64cd1ec87b6ab593ba3d2ab07da43e2;p=thirdparty%2Fccache.git

Add --evict-older-than (#605)

The argument adds another mechanism to control contents of cache directory. And
is based on the LRU tracking behaviour.

As of now there is no way for ccache to eliminate files which were
are no longer needed. As a result these files stay and are kept around until
either max_files/max_size is reached.

If a particular project is being built regularly but for some reason is allowed
to grow in size, then under such circumstances using the LRU mechanism to control
cache size, lends as perfect solution.

The argument takes a parameter N and performs a cleanup.
While performing cleanup the oldest file in ccache can
only be N seconds old. However this cleanup will not take max_files and
max_old into consideration
---

diff --git a/doc/MANUAL.adoc b/doc/MANUAL.adoc
index 7cd035c71..c4bbad56f 100644
--- a/doc/MANUAL.adoc
+++ b/doc/MANUAL.adoc
@@ -94,6 +94,16 @@ Common options
     Clear the entire cache, removing all cached files, but keeping the
     configuration file.
 
+*`--evict-older-than`* _NUM_::
+
+    This is another cleanup mechanism (independent of max_files and max_size).
+    It instructs ccache to perform cleanup based on age of the stored files.
+    The value _NUM_ is duration and means any cache entry older than it from CURRENT_TIME
+    should be deleted. _NUM_ should be an unsigned number with a suffix:
+    d(ays)/s(econds).
+    This cleanup can be performed from the cli with this option only. During the
+    cleanup max_files/max_size will not be taken into consideration.
+
 *`-h`*, *`--help`*::
 
     Print an options summary page.
diff --git a/src/Util.cpp b/src/Util.cpp
index cef42b20c..b20c549a1 100644
--- a/src/Util.cpp
+++ b/src/Util.cpp
@@ -976,4 +976,37 @@ write_file(const std::string& path,
   file << data;
 }
 
+unsigned
+parse_duration_with_suffix_to_seconds(const std::string& value)
+{
+  size_t end;
+  long result;
+  bool failed = false;
+
+  try {
+    result = std::stol(value, &end, 10);
+  } catch (std::exception&) {
+    failed = true;
+  }
+
+  if (failed || result < 0) {
+    throw Error(fmt::format("invalid unsigned integer: \"{}\"", value));
+  }
+
+  if (end + 1 != value.size()) {
+    throw Error(
+      fmt::format("Invalid suffix, Supported: d(ay)/s(econd): \"{}\"", value));
+  }
+
+  switch (value[end]) {
+  case 'd':
+    result *= 24 * 3600;
+  case 's':
+    break;
+  default:
+    throw Error(
+      fmt::format("Invalid suffix, Supported: d(ay)/s(econd): \"{}\"", value));
+  }
+  return result;
+}
 } // namespace Util
diff --git a/src/Util.hpp b/src/Util.hpp
index 4492fc846..10f5fdb8a 100644
--- a/src/Util.hpp
+++ b/src/Util.hpp
@@ -347,4 +347,11 @@ void write_file(const std::string& path,
                 const std::string& data,
                 std::ios_base::openmode open_mode = std::ios::binary);
 
+// Parse the given string into an unsigned integer. Then based on suffix
+// provided convert the number to seconds possible suffixes = d(ays)/s(econds)
+//
+// Throws `Error` for any other suffix
+// Throws `Error` if parse value is <0
+unsigned parse_duration_with_suffix_to_seconds(const std::string& value);
+
 } // namespace Util
diff --git a/src/ccache.cpp b/src/ccache.cpp
index 057f50e9d..c12c6a854 100644
--- a/src/ccache.cpp
+++ b/src/ccache.cpp
@@ -89,6 +89,10 @@ Common options:
                               (normally not needed as this is done
                               automatically)
     -C, --clear               clear the cache completely (except configuration)
+        --evict-older-than N  delete files older than N (days/seconds) (this will not
+                              take max_files, max_size into consideration).
+                              N should be an unsigned number with a suffix:
+                              d(ays)/s(econds).
     -F, --max-files NUM       set maximum number of files in cache to NUM (use 0
                               for no limit)
     -M, --max-size SIZE       set maximum size of cache to SIZE (use 0 for no
@@ -2200,6 +2204,7 @@ handle_main_options(int argc, const char* const* argv)
     DUMP_MANIFEST,
     DUMP_RESULT,
     EXTRACT_RESULT,
+    EVICT_OLDER_THAN,
     HASH_FILE,
     PRINT_STATS,
   };
@@ -2209,6 +2214,7 @@ handle_main_options(int argc, const char* const* argv)
     {"dump-manifest", required_argument, nullptr, DUMP_MANIFEST},
     {"dump-result", required_argument, nullptr, DUMP_RESULT},
     {"extract-result", required_argument, nullptr, EXTRACT_RESULT},
+    {"evict-older-than", required_argument, nullptr, EVICT_OLDER_THAN},
     {"get-config", required_argument, nullptr, 'k'},
     {"hash-file", required_argument, nullptr, HASH_FILE},
     {"help", no_argument, nullptr, 'h'},
@@ -2258,6 +2264,17 @@ handle_main_options(int argc, const char* const* argv)
       return error ? EXIT_FAILURE : EXIT_SUCCESS;
     }
 
+    case EVICT_OLDER_THAN: {
+      unsigned seconds = Util::parse_duration_with_suffix_to_seconds(optarg);
+      ProgressBar progress_bar("Clearing ...");
+      clean_old(
+        ctx, [&](double progress) { progress_bar.update(progress); }, seconds);
+      if (isatty(STDOUT_FILENO)) {
+        printf("\n");
+      }
+      break;
+    }
+
     case HASH_FILE: {
       Hash hash;
       if (str_eq(optarg, "-")) {
diff --git a/src/cleanup.cpp b/src/cleanup.cpp
index 8423301d5..1e41cde11 100644
--- a/src/cleanup.cpp
+++ b/src/cleanup.cpp
@@ -57,6 +57,7 @@ void
 clean_up_dir(const std::string& subdir,
              uint64_t max_size,
              uint32_t max_files,
+             time_t max_age,
              const Util::ProgressReceiver& progress_receiver)
 {
   cc_log("Cleaning up cache directory %s", subdir.c_str());
@@ -111,7 +112,8 @@ clean_up_dir(const std::string& subdir,
     }
 
     if ((max_size == 0 || cache_size <= max_size)
-        && (max_files == 0 || files_in_cache <= max_files)) {
+        && (max_files == 0 || files_in_cache <= max_files)
+        && (max_age == 0 || file->lstat().mtime() > (current_time - max_age))) {
       break;
     }
 
@@ -163,6 +165,7 @@ clean_up_all(const Config& config,
       clean_up_dir(subdir,
                    config.max_size() / 16,
                    config.max_files() / 16,
+                   0,
                    sub_progress_receiver);
     },
     progress_receiver);
@@ -202,3 +205,17 @@ wipe_all(const Context& ctx, const Util::ProgressReceiver& progress_receiver)
   ctx.inode_cache.drop();
 #endif
 }
+
+void
+clean_old(const Context& ctx,
+          const Util::ProgressReceiver& progress_receiver,
+          time_t max_age)
+{
+  Util::for_each_level_1_subdir(
+    ctx.config.cache_dir(),
+    [&](const std::string& subdir,
+        const Util::ProgressReceiver& sub_progress_receiver) {
+      clean_up_dir(subdir, 0, 0, max_age, sub_progress_receiver);
+    },
+    progress_receiver);
+}
diff --git a/src/cleanup.hpp b/src/cleanup.hpp
index 1e34e66d4..5148bf91e 100644
--- a/src/cleanup.hpp
+++ b/src/cleanup.hpp
@@ -30,6 +30,7 @@ class Context;
 void clean_up_dir(const std::string& subdir,
                   uint64_t max_size,
                   uint32_t max_files,
+                  time_t max_age,
                   const Util::ProgressReceiver& progress_receiver);
 
 void clean_up_all(const Config& config,
@@ -37,3 +38,7 @@ void clean_up_all(const Config& config,
 
 void wipe_all(const Context& ctx,
               const Util::ProgressReceiver& progress_receiver);
+
+void clean_old(const Context& ctx,
+               const Util::ProgressReceiver& progress_receiver,
+               time_t max_age);
diff --git a/src/stats.cpp b/src/stats.cpp
index ec3979572..4fe1966f4 100644
--- a/src/stats.cpp
+++ b/src/stats.cpp
@@ -372,7 +372,9 @@ stats_flush_to_file(const Config& config,
     double factor = config.limit_multiple() / 16;
     uint64_t max_size = round(config.max_size() * factor);
     uint32_t max_files = round(config.max_files() * factor);
-    clean_up_dir(subdir, max_size, max_files, [](double /*progress*/) {});
+    uint32_t max_age = 0;
+    clean_up_dir(
+      subdir, max_size, max_files, max_age, [](double /*progress*/) {});
   }
 }
 
diff --git a/test/suites/cleanup.bash b/test/suites/cleanup.bash
index af2f4f7af..a85088c79 100644
--- a/test/suites/cleanup.bash
+++ b/test/suites/cleanup.bash
@@ -179,4 +179,22 @@ SUITE_cleanup() {
     $CCACHE -c >/dev/null
     expect_file_count 1 '.nfs*' $CCACHE_DIR
     expect_stat 'files in cache' 10
+    # -------------------------------------------------------------------------
+    TEST "cleanup of old files by age"
+
+    prepare_cleanup_test_dir $CCACHE_DIR/a
+    touch $CCACHE_DIR/a/now.result
+    $CCACHE -F 0 -M 0 >/dev/null
+
+    $CCACHE --evict-older-than 1d >/dev/null
+    expect_file_count 1 '*.result' $CCACHE_DIR
+    expect_stat 'files in cache' 1
+
+    $CCACHE --evict-older-than 1d  >/dev/null
+    expect_file_count 1 '*.result' $CCACHE_DIR
+    expect_stat 'files in cache' 1
+
+    backdate $CCACHE_DIR/a/now.result
+    $CCACHE --evict-older-than 10s  >/dev/null
+    expect_stat 'files in cache' 0
 }
diff --git a/unittest/test_Util.cpp b/unittest/test_Util.cpp
index ce4638af0..71586e07f 100644
--- a/unittest/test_Util.cpp
+++ b/unittest/test_Util.cpp
@@ -766,4 +766,18 @@ TEST_CASE("Util::wipe_path")
   }
 }
 
+TEST_CASE("Util::parse_duration_with_suffix_to_seconds")
+{
+  CHECK(Util::parse_duration_with_suffix_to_seconds("0s") == 0);
+  CHECK(Util::parse_duration_with_suffix_to_seconds("2s") == 2);
+  CHECK(Util::parse_duration_with_suffix_to_seconds("1d") == 3600 * 24);
+  CHECK(Util::parse_duration_with_suffix_to_seconds("2d") == 2 * 3600 * 24);
+  CHECK_THROWS_WITH(Util::parse_duration_with_suffix_to_seconds("-2"),
+                    "invalid unsigned integer: \"-2\"");
+  CHECK_THROWS_WITH(Util::parse_duration_with_suffix_to_seconds("2x"),
+                    "Invalid suffix, Supported: d(ay)/s(econd): \"2x\"");
+  CHECK_THROWS_WITH(Util::parse_duration_with_suffix_to_seconds("2"),
+                    "Invalid suffix, Supported: d(ay)/s(econd): \"2\"");
+}
+
 TEST_SUITE_END();