From: Joel Rosdahl Date: Tue, 10 Aug 2021 15:44:30 +0000 (+0200) Subject: feat: Add --trim-dir option for usage with secondary storage directories X-Git-Tag: v4.4~28 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a358033b60fa4e5dbc92ff3cb1415524ba89c181;p=thirdparty%2Fccache.git feat: Add --trim-dir option for usage with secondary storage directories --- diff --git a/doc/MANUAL.adoc b/doc/MANUAL.adoc index be679864d..a1a1532d6 100644 --- a/doc/MANUAL.adoc +++ b/doc/MANUAL.adoc @@ -169,6 +169,35 @@ compiler's documentation. Zero the cache statistics (but not the configuration options). +=== Options for secondary storage + +*--trim-dir* _PATH_:: + + Remove old files from directory _PATH_ until it is at most the size specified + by `--trim-max-size`. ++ +WARNING: Don't use this option to trim the primary cache. To trim the primary +cache directory to a certain size, use `CCACHE_MAXSIZE=_SIZE_ ccache -c`. + +*--trim-max-size* _SIZE_:: + + Specify the maximum size for `--trim-dir`. _SIZE_ should be a number followed + by an optional suffix: k, M, G, T (decimal), Ki, Mi, Gi or Ti (binary). The + default suffix is G. + +*--trim-method* _METHOD_:: + + Specify the method to trim a directory with `--trim-dir`. Possible values + are: ++ +-- +*atime*:: + LRU (least recently used) using the file access timestamp. This is the + default. +*mtime*:: + LRU (least recently used) using the file modification timestamp. +-- + === Options for scripting or debugging *--checksum-file* _PATH_:: @@ -996,8 +1025,10 @@ URL format: `+file:DIRECTORY+` or `+file://DIRECTORY+` This backend stores data as separate files in a directory structure below *DIRECTORY* (an absolute path), similar (but not identical) to the primary cache storage. A typical use case for this backend would be sharing a cache on an NFS -directory. Note that ccache will not perform any cleanup of the storage -- that -has to be done by other means. +directory. + +IMPORTANT: ccache will not perform any cleanup of the storage -- that has to be +done by other means, for instance by running `ccache --trim-dir` periodically. Examples: @@ -1027,7 +1058,8 @@ URL format: `+http://HOST[:PORT][/PATH]+` This backend stores data in an HTTP-compatible server. The required HTTP methods are `GET`, `PUT` and `DELETE`. -IMPORTANT: ccache will not perform any cleanup of the HTTP storage. +IMPORTANT: ccache will not perform any cleanup of the storage -- that has to be +done by other means, for instance by running `ccache --trim-dir` periodically. NOTE: HTTPS is not supported. diff --git a/src/Stat.hpp b/src/Stat.hpp index b6e84e0da..2f56214a5 100644 --- a/src/Stat.hpp +++ b/src/Stat.hpp @@ -126,6 +126,7 @@ public: dev_t device() const; ino_t inode() const; mode_t mode() const; + time_t atime() const; time_t ctime() const; time_t mtime() const; uint64_t size() const; @@ -141,6 +142,7 @@ public: uint32_t reparse_tag() const; #endif + timespec atim() const; timespec ctim() const; timespec mtim() const; @@ -196,6 +198,12 @@ Stat::mode() const return m_stat.st_mode; } +inline time_t +Stat::atime() const +{ + return atim().tv_sec; +} + inline time_t Stat::ctime() const { @@ -256,6 +264,16 @@ Stat::reparse_tag() const } #endif +inline timespec +Stat::atim() const +{ +#if defined(_WIN32) || defined(HAVE_STRUCT_STAT_ST_ATIM) + return m_stat.st_atim; +#else + return {m_stat.st_atime, 0}; +#endif +} + inline timespec Stat::ctim() const { diff --git a/src/core/mainoptions.cpp b/src/core/mainoptions.cpp index dc1070a48..f95c9e78f 100644 --- a/src/core/mainoptions.cpp +++ b/src/core/mainoptions.cpp @@ -113,6 +113,17 @@ Common options: -h, --help print this help text -V, --version print version and copyright information +Options for secondary storage: + --trim-dir PATH remove old files from directory _PATH_ until it + is at most the size specified by --trim-max-size + (note: don't use this option to trim the primary + cache) + --trim-max-size SIZE specify the maximum size for --trim-dir; + available suffixes: k, M, G, T (decimal) and Ki, + Mi, Gi, Ti (binary); default suffix: G + --trim-method METHOD specify the method (atime or mtime) for + --trim-dir; default: atime + Options for scripting or debugging: --checksum-file PATH print the checksum (64 bit XXH3) of the file at PATH @@ -176,6 +187,61 @@ print_compression_statistics(const storage::primary::CompressionStatistics& cs) PRINT_RAW(stdout, table.render()); } +static void +trim_dir(const std::string& dir, + const uint64_t trim_max_size, + const bool trim_lru_mtime) +{ + struct File + { + std::string path; + Stat stat; + }; + std::vector files; + uint64_t size_before = 0; + + Util::traverse(dir, [&](const std::string& path, const bool is_dir) { + const auto stat = Stat::lstat(path); + if (!stat) { + // Probably some race, ignore. + return; + } + size_before += stat.size_on_disk(); + if (!is_dir) { + const auto name = Util::base_name(path); + if (name == "ccache.conf" || name == "stats") { + throw Fatal("this looks like a primary cache directory (found {})", + path); + } + files.push_back({path, stat}); + } + }); + + std::sort(files.begin(), files.end(), [&](const auto& f1, const auto& f2) { + if (trim_lru_mtime) { + return f1.stat.mtime() < f2.stat.mtime(); + } else { + return f1.stat.atime() < f2.stat.atime(); + } + }); + + uint64_t size_after = size_before; + + for (const auto& file : files) { + if (size_after <= trim_max_size) { + break; + } + Util::unlink_tmp(file.path); + size_after -= file.stat.size(); + } + + PRINT(stdout, + "Removed {} ({} -> {})\n", + Util::format_human_readable_size(size_before - size_after), + Util::format_human_readable_size(size_before), + Util::format_human_readable_size(size_after)); +} + static std::string get_version_text() { @@ -199,6 +265,9 @@ enum { HASH_FILE, PRINT_STATS, SHOW_LOG_STATS, + TRIM_DIR, + TRIM_MAX_SIZE, + TRIM_METHOD, }; const char options_string[] = "cCd:k:hF:M:po:sVxX:z"; @@ -224,6 +293,9 @@ const option long_options[] = { {"show-config", no_argument, nullptr, 'p'}, {"show-log-stats", no_argument, nullptr, SHOW_LOG_STATS}, {"show-stats", no_argument, nullptr, 's'}, + {"trim-dir", required_argument, nullptr, TRIM_DIR}, + {"trim-max-size", required_argument, nullptr, TRIM_MAX_SIZE}, + {"trim-method", required_argument, nullptr, TRIM_METHOD}, {"version", no_argument, nullptr, 'V'}, {"zero-stats", no_argument, nullptr, 'z'}, {nullptr, 0, nullptr, 0}}; @@ -232,6 +304,8 @@ int process_main_options(int argc, const char* const* argv) { int c; + nonstd::optional trim_max_size; + bool trim_lru_mtime = false; // First pass: Handle non-command options that affect command options. while ((c = getopt_long(argc, @@ -240,13 +314,23 @@ process_main_options(int argc, const char* const* argv) long_options, nullptr)) != -1) { + const std::string arg = optarg ? optarg : std::string(); + switch (c) { case 'd': // --directory - Util::setenv("CCACHE_DIR", optarg); + Util::setenv("CCACHE_DIR", arg); break; case CONFIG_PATH: - Util::setenv("CCACHE_CONFIGPATH", optarg); + Util::setenv("CCACHE_CONFIGPATH", arg); + break; + + case TRIM_MAX_SIZE: + trim_max_size = Util::parse_size(arg); + break; + + case TRIM_METHOD: + trim_lru_mtime = (arg == "ctime"); break; } } @@ -262,14 +346,15 @@ process_main_options(int argc, const char* const* argv) Config config; config.read(); - std::string arg = optarg ? optarg : std::string(); + const std::string arg = optarg ? optarg : std::string(); switch (c) { case CONFIG_PATH: - break; // Already handled in the first pass. - case 'd': // --directory - break; // Already handled in the first pass. + case TRIM_MAX_SIZE: + case TRIM_METHOD: + // Already handled in the first pass. + break; case CHECKSUM_FILE: { Checksum checksum; @@ -434,6 +519,13 @@ process_main_options(int argc, const char* const* argv) break; } + case TRIM_DIR: + if (!trim_max_size) { + throw Error("please specify --trim-max-size when using --trim-dir"); + } + trim_dir(arg, *trim_max_size, trim_lru_mtime); + break; + case 'V': // --version PRINT_RAW(stdout, get_version_text()); exit(EXIT_SUCCESS); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 20ccabfdc..fa8feb3ff 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -60,4 +60,5 @@ addtest(serialize_diagnostics) addtest(source_date_epoch) addtest(split_dwarf) addtest(stats_log) +addtest(trim_dir) addtest(upgrade) diff --git a/test/suites/trim_dir.bash b/test/suites/trim_dir.bash new file mode 100644 index 000000000..1edb119fb --- /dev/null +++ b/test/suites/trim_dir.bash @@ -0,0 +1,41 @@ +SUITE_trim_dir() { + # ------------------------------------------------------------------------- + TEST "Trim secondary cache directory" + + if $HOST_OS_APPLE; then + one_mb=1m + else + one_mb=1M + for subdir in aa bb cc; do + mkdir -p secondary/$subdir + dd if=/dev/zero of=secondary/$subdir/1 count=1 bs=$one_mb 2/dev/null + dd if=/dev/zero of=secondary/$subdir/2 count=1 bs=$one_mb 2/dev/null + done + + backdate secondary/bb/2 secondary/cc/1 + $CCACHE --trim-dir secondary --trim-max-size 4.5M --trim-method mtime \ + >/dev/null + + expect_exists secondary/aa/1 + expect_exists secondary/aa/2 + expect_exists secondary/bb/1 + expect_missing secondary/bb/2 + expect_missing secondary/cc/1 + expect_exists secondary/cc/2 + + # ------------------------------------------------------------------------- + TEST "Trim primary cache directory" + + mkdir -p primary/0 + touch primary/0/stats + if $CCACHE --trim-dir primary --trim-max-size 0 &>/dev/null; then + test_failed "Expected failure" + fi + + rm -rf primary + mkdir primary + touch primary/ccache.conf + if $CCACHE --trim-dir primary --trim-max-size 0 &>/dev/null; then + test_failed "Expected failure" + fi +}