From: Joel Rosdahl Date: Mon, 8 Nov 2021 18:38:35 +0000 (+0100) Subject: feat: Add namespace support X-Git-Tag: v4.5~4 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=9217417c804b811bd33ecae554919b3db78fd4e9;p=thirdparty%2Fccache.git feat: Add namespace support If a namespace configured, the namespace string will be added to the hashed data for each compilation. This will make the associated cache entries logically separate from cache entries with other namespaces, but they will still share the same storage space. Cache entries can also be selectively removed from the primary cache with the command line option --evict-namespace, potentially in combination with --evict-older-than. Closes #870. --- diff --git a/doc/MANUAL.adoc b/doc/MANUAL.adoc index 033dd0b71..1577fb4e7 100644 --- a/doc/MANUAL.adoc +++ b/doc/MANUAL.adoc @@ -99,10 +99,16 @@ compiler's documentation. has the same effect as setting the environment variable `CCACHE_DIR` temporarily. +*--evict-namespace* _NAMESPACE_:: + + Remove files created with a certain <> from + the cache. + *--evict-older-than* _AGE_:: Remove files older than _AGE_ from the cache. _AGE_ should be an unsigned - integer with a `d` (days) or `s` (seconds) suffix. + integer with a `d` (days) or `s` (seconds) suffix. If combined with + `--evict-namespace`, only remove old files within that namespace. *-h*, *--help*:: @@ -761,6 +767,21 @@ file in `/etc/rsyslog.d`: Gi, Ti (binary). The default suffix is G. See also _<>_. +[#config_namespace] +*namespace* (*CCACHE_NAMESPACE*):: + + If set, the namespace string will be added to the hashed data for each + compilation. This will make the associated cache entries logically separate + from cache entries with other namespaces, but they will still share the same + storage space. Cache entries can also be selectively removed from the + primary cache with the command line option `--evict-namespace`, potentially + in combination with `--evict-older-than`. +. +For instance, if you use the same primary cache for several disparate projects, +you can use a unique namespace string for each one. This allows you to remove +cache entries that belong to a certain project if stop working with that +project. + [#config_path] *path* (*CCACHE_PATH*):: diff --git a/src/Config.cpp b/src/Config.cpp index ec739c8af..9b619b211 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -82,6 +82,7 @@ enum class ConfigItem { log_file, max_files, max_size, + namespace_, path, pch_external_checksum, prefix_command, @@ -126,6 +127,7 @@ const std::unordered_map k_config_key_table = { {"log_file", ConfigItem::log_file}, {"max_files", ConfigItem::max_files}, {"max_size", ConfigItem::max_size}, + {"namespace", ConfigItem::namespace_}, {"path", ConfigItem::path}, {"pch_external_checksum", ConfigItem::pch_external_checksum}, {"prefix_command", ConfigItem::prefix_command}, @@ -172,6 +174,7 @@ const std::unordered_map k_env_variable_table = { {"LOGFILE", "log_file"}, {"MAXFILES", "max_files"}, {"MAXSIZE", "max_size"}, + {"NAMESPACE", "namespace"}, {"PATH", "path"}, {"PCH_EXTSUM", "pch_external_checksum"}, {"PREFIX", "prefix_command"}, @@ -687,6 +690,9 @@ Config::get_string_value(const std::string& key) const case ConfigItem::max_size: return format_cache_size(m_max_size); + case ConfigItem::namespace_: + return m_namespace; + case ConfigItem::path: return m_path; @@ -927,6 +933,10 @@ Config::set_item(const std::string& key, m_max_size = Util::parse_size(value); break; + case ConfigItem::namespace_: + m_namespace = Util::expand_environment_variables(value); + break; + case ConfigItem::path: m_path = Util::expand_environment_variables(value); break; diff --git a/src/Config.hpp b/src/Config.hpp index 9383106bd..4e386870d 100644 --- a/src/Config.hpp +++ b/src/Config.hpp @@ -81,6 +81,7 @@ public: core::Sloppiness sloppiness() const; bool stats() const; const std::string& stats_log() const; + const std::string& namespace_() const; const std::string& temporary_dir() const; nonstd::optional umask() const; @@ -178,6 +179,7 @@ private: core::Sloppiness m_sloppiness; bool m_stats = true; std::string m_stats_log; + std::string m_namespace; std::string m_temporary_dir; nonstd::optional m_umask; @@ -428,6 +430,12 @@ Config::stats_log() const return m_stats_log; } +inline const std::string& +Config::namespace_() const +{ + return m_namespace; +} + inline const std::string& Config::temporary_dir() const { diff --git a/src/Manifest.cpp b/src/Manifest.cpp index 09d7683a0..341e66dbf 100644 --- a/src/Manifest.cpp +++ b/src/Manifest.cpp @@ -341,7 +341,7 @@ write_manifest(const Config& config, compression::level_from_config(config), time(nullptr), CCACHE_VERSION, - ""); + config.namespace_()); header.set_entry_size_from_payload_size(payload_size); core::CacheEntryWriter writer(file_writer, header); diff --git a/src/Result.cpp b/src/Result.cpp index e79334f34..b36cd231b 100644 --- a/src/Result.cpp +++ b/src/Result.cpp @@ -342,7 +342,7 @@ Writer::do_finalize() compression::level_from_config(m_ctx.config), time(nullptr), CCACHE_VERSION, - ""); + m_ctx.config.namespace_()); header.set_entry_size_from_payload_size(payload_size); core::FileWriter file_writer(atomic_result_file.stream()); diff --git a/src/ccache.cpp b/src/ccache.cpp index 406ae738e..85cc59a88 100644 --- a/src/ccache.cpp +++ b/src/ccache.cpp @@ -1189,6 +1189,11 @@ hash_common_info(const Context& ctx, { hash.hash(HASH_PREFIX); + if (!ctx.config.namespace_().empty()) { + hash.hash_delimiter("namespace"); + hash.hash(ctx.config.namespace_()); + } + // We have to hash the extension, as a .i file isn't treated the same by the // compiler as a .ii file. hash.hash_delimiter("ext"); diff --git a/src/core/mainoptions.cpp b/src/core/mainoptions.cpp index 7bb3c0bc5..1876ce93f 100644 --- a/src/core/mainoptions.cpp +++ b/src/core/mainoptions.cpp @@ -90,6 +90,8 @@ Common options: default -d, --dir PATH operate on cache directory PATH instead of the default + --evict-namespace NAMESPACE + remove files created with namespace NAMESPACE --evict-older-than AGE remove files older than AGE (unsigned integer with a d (days) or s (seconds) suffix) -F, --max-files NUM set maximum number of files in cache to NUM (use @@ -261,6 +263,7 @@ enum { CONFIG_PATH, DUMP_MANIFEST, DUMP_RESULT, + EVICT_NAMESPACE, EVICT_OLDER_THAN, EXTRACT_RESULT, HASH_FILE, @@ -281,6 +284,7 @@ const option long_options[] = { {"directory", required_argument, nullptr, 'd'}, // backward compatibility {"dump-manifest", required_argument, nullptr, DUMP_MANIFEST}, {"dump-result", required_argument, nullptr, DUMP_RESULT}, + {"evict-namespace", required_argument, nullptr, EVICT_NAMESPACE}, {"evict-older-than", required_argument, nullptr, EVICT_OLDER_THAN}, {"extract-result", required_argument, nullptr, EXTRACT_RESULT}, {"get-config", required_argument, nullptr, 'k'}, @@ -310,6 +314,8 @@ process_main_options(int argc, const char* const* argv) nonstd::optional trim_max_size; bool trim_lru_mtime = false; uint8_t verbosity = 0; + nonstd::optional evict_namespace; + nonstd::optional evict_max_age; // First pass: Handle non-command options that affect command options. while ((c = getopt_long(argc, @@ -392,14 +398,13 @@ process_main_options(int argc, const char* const* argv) return error ? EXIT_FAILURE : EXIT_SUCCESS; } + case EVICT_NAMESPACE: { + evict_namespace = arg; + break; + } + case EVICT_OLDER_THAN: { - auto seconds = Util::parse_duration(arg); - ProgressBar progress_bar("Evicting..."); - storage::primary::PrimaryStorage(config).clean_old( - [&](double progress) { progress_bar.update(progress); }, seconds); - if (isatty(STDOUT_FILENO)) { - PRINT_RAW(stdout, "\n"); - } + evict_max_age = Util::parse_duration(arg); break; } @@ -590,6 +595,20 @@ process_main_options(int argc, const char* const* argv) } } + if (evict_max_age || evict_namespace) { + Config config; + config.read(); + + ProgressBar progress_bar("Evicting..."); + storage::primary::PrimaryStorage(config).evict( + [&](double progress) { progress_bar.update(progress); }, + evict_max_age, + evict_namespace); + if (isatty(STDOUT_FILENO)) { + PRINT_RAW(stdout, "\n"); + } + } + return EXIT_SUCCESS; } diff --git a/src/storage/primary/CacheFile.cpp b/src/storage/primary/CacheFile.cpp index 8ada17ff7..0068fcdd7 100644 --- a/src/storage/primary/CacheFile.cpp +++ b/src/storage/primary/CacheFile.cpp @@ -39,6 +39,8 @@ CacheFile::type() const return Type::manifest; } else if (util::ends_with(m_path, Result::k_file_suffix)) { return Type::result; + } else if (util::ends_with(m_path, "W")) { + return Type::raw; } else { return Type::unknown; } diff --git a/src/storage/primary/CacheFile.hpp b/src/storage/primary/CacheFile.hpp index 14064fec4..809ea38a4 100644 --- a/src/storage/primary/CacheFile.hpp +++ b/src/storage/primary/CacheFile.hpp @@ -27,7 +27,7 @@ class CacheFile { public: - enum class Type { result, manifest, unknown }; + enum class Type { result, manifest, raw, unknown }; explicit CacheFile(const std::string& path); diff --git a/src/storage/primary/PrimaryStorage.cpp b/src/storage/primary/PrimaryStorage.cpp index bbe4d53f1..a4280b345 100644 --- a/src/storage/primary/PrimaryStorage.cpp +++ b/src/storage/primary/PrimaryStorage.cpp @@ -173,8 +173,12 @@ PrimaryStorage::finalize() const double factor = m_config.limit_multiple() / 16; const uint64_t max_size = round(m_config.max_size() * factor); const uint32_t max_files = round(m_config.max_files() * factor); - const time_t max_age = 0; - clean_dir(subdir, max_size, max_files, max_age, [](double /*progress*/) {}); + clean_dir(subdir, + max_size, + max_files, + nonstd::nullopt, + nonstd::nullopt, + [](double /*progress*/) {}); } } diff --git a/src/storage/primary/PrimaryStorage.hpp b/src/storage/primary/PrimaryStorage.hpp index 99f47a598..bb8b50545 100644 --- a/src/storage/primary/PrimaryStorage.hpp +++ b/src/storage/primary/PrimaryStorage.hpp @@ -77,7 +77,9 @@ public: // --- Cleanup --- - void clean_old(const ProgressReceiver& progress_receiver, uint64_t max_age); + void evict(const ProgressReceiver& progress_receiver, + nonstd::optional max_age, + nonstd::optional namespace_); void clean_all(const ProgressReceiver& progress_receiver); @@ -137,7 +139,8 @@ private: static void clean_dir(const std::string& subdir, uint64_t max_size, uint64_t max_files, - uint64_t max_age, + nonstd::optional max_age, + nonstd::optional namespace_, const ProgressReceiver& progress_receiver); }; diff --git a/src/storage/primary/PrimaryStorage_cleanup.cpp b/src/storage/primary/PrimaryStorage_cleanup.cpp index 6f3790f9c..63b956ac4 100644 --- a/src/storage/primary/PrimaryStorage_cleanup.cpp +++ b/src/storage/primary/PrimaryStorage_cleanup.cpp @@ -21,8 +21,12 @@ #include #include +#include #include #include +#include +#include +#include #include #include #include @@ -75,14 +79,15 @@ update_counters(const std::string& dir, } void -PrimaryStorage::clean_old(const ProgressReceiver& progress_receiver, - const uint64_t max_age) +PrimaryStorage::evict(const ProgressReceiver& progress_receiver, + nonstd::optional max_age, + nonstd::optional namespace_) { for_each_level_1_subdir( m_config.cache_dir(), [&](const std::string& subdir, const ProgressReceiver& sub_progress_receiver) { - clean_dir(subdir, 0, 0, max_age, sub_progress_receiver); + clean_dir(subdir, 0, 0, max_age, namespace_, sub_progress_receiver); }, progress_receiver); } @@ -92,7 +97,8 @@ void PrimaryStorage::clean_dir(const std::string& subdir, const uint64_t max_size, const uint64_t max_files, - const uint64_t max_age, + const nonstd::optional max_age, + const nonstd::optional namespace_, const ProgressReceiver& progress_receiver) { LOG("Cleaning up cache directory {}", subdir); @@ -103,6 +109,9 @@ PrimaryStorage::clean_dir(const std::string& subdir, uint64_t cache_size = 0; uint64_t files_in_cache = 0; time_t current_time = time(nullptr); + std::unordered_map /*associated_raw_files*/> + raw_files_map; for (size_t i = 0; i < files.size(); ++i, progress_receiver(1.0 / 3 + 1.0 * i / files.size() / 3)) { @@ -120,6 +129,12 @@ PrimaryStorage::clean_dir(const std::string& subdir, continue; } + if (namespace_ && file.type() == CacheFile::Type::raw) { + const auto result_filename = + FMT("{}R", file.path().substr(0, file.path().length() - 2)); + raw_files_map[result_filename].push_back(file.path()); + } + cache_size += file.lstat().size_on_disk(); files_in_cache += 1; } @@ -148,12 +163,41 @@ PrimaryStorage::clean_dir(const std::string& subdir, if ((max_size == 0 || cache_size <= max_size) && (max_files == 0 || files_in_cache <= max_files) - && (max_age == 0 + && (!max_age || file.lstat().mtime() - > (current_time - static_cast(max_age)))) { + > (current_time - static_cast(*max_age))) + && (!namespace_ || max_age)) { break; } + if (namespace_) { + try { + File file_stream(file.path(), "rb"); + core::FileReader file_reader(*file_stream); + core::CacheEntryReader reader(file_reader); + if (reader.header().namespace_ != *namespace_) { + continue; + } + } catch (core::Error&) { + // Failed to read header: ignore. + continue; + } + + // For namespace eviction we need to remove raw files based on result + // filename since they don't have a header. + if (file.type() == CacheFile::Type::result) { + const auto entry = raw_files_map.find(file.path()); + if (entry != raw_files_map.end()) { + for (const auto& raw_file : entry->second) { + delete_file(raw_file, + Stat::lstat(raw_file).size_on_disk(), + &cache_size, + &files_in_cache); + } + } + } + } + if (util::ends_with(file.path(), ".stderr")) { // In order to be nice to legacy ccache versions, make sure that the .o // file is deleted before .stderr, because if the ccache process gets @@ -199,7 +243,8 @@ PrimaryStorage::clean_all(const ProgressReceiver& progress_receiver) clean_dir(subdir, m_config.max_size() / 16, m_config.max_files() / 16, - 0, + nonstd::nullopt, + nonstd::nullopt, sub_progress_receiver); }, progress_receiver); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 0015e4117..42d4c22fe 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -39,6 +39,7 @@ addtest(ivfsoverlay) addtest(masquerading) addtest(modules) addtest(multi_arch) +addtest(namespace) addtest(no_compression) addtest(nocpp2) addtest(nvcc) diff --git a/test/suites/namespace.bash b/test/suites/namespace.bash new file mode 100644 index 000000000..accf4001d --- /dev/null +++ b/test/suites/namespace.bash @@ -0,0 +1,58 @@ +SUITE_namespace_SETUP() { + unset CCACHE_NODIRECT + echo 'int x;' >test1.c + echo 'int y;' >test2.c +} + +SUITE_namespace() { + # ------------------------------------------------------------------------- + TEST "Namespace makes entries isolated" + + $CCACHE_COMPILE -c test1.c + expect_stat direct_cache_hit 0 + expect_stat cache_miss 1 + + $CCACHE_COMPILE -c test1.c + expect_stat direct_cache_hit 1 + expect_stat cache_miss 1 + + CCACHE_NAMESPACE=a $CCACHE_COMPILE -c test1.c + expect_stat direct_cache_hit 1 + expect_stat cache_miss 2 + + CCACHE_NAMESPACE=a $CCACHE_COMPILE -c test1.c + expect_stat direct_cache_hit 2 + expect_stat cache_miss 2 + + CCACHE_NAMESPACE=b $CCACHE_COMPILE -c test1.c + expect_stat direct_cache_hit 2 + expect_stat cache_miss 3 + + CCACHE_NAMESPACE=b $CCACHE_COMPILE -c test1.c + expect_stat direct_cache_hit 3 + expect_stat cache_miss 3 + + # ------------------------------------------------------------------------- + TEST "--evict-namespace + --evict-older-than" + + CCACHE_NAMESPACE="a" $CCACHE_COMPILE -c test1.c + result_file="$(find $CCACHE_DIR -name '*R')" + backdate "$result_file" + for ns in a b c; do + CCACHE_NAMESPACE="$ns" $CCACHE_COMPILE -c test2.c + done + expect_stat cache_miss 4 + expect_stat files_in_cache 8 + + $CCACHE --evict-namespace d >/dev/null + expect_stat files_in_cache 8 + + $CCACHE --evict-namespace c >/dev/null + expect_stat files_in_cache 6 + + $CCACHE --evict-namespace a --evict-older-than 1d >/dev/null + expect_stat files_in_cache 5 + + $CCACHE --evict-namespace a >/dev/null + expect_stat files_in_cache 2 +} diff --git a/unittest/test_Config.cpp b/unittest/test_Config.cpp index 3b62174c4..17a055d57 100644 --- a/unittest/test_Config.cpp +++ b/unittest/test_Config.cpp @@ -402,6 +402,7 @@ TEST_CASE("Config::visit_items") "log_file = lf\n" "max_files = 4711\n" "max_size = 98.7M\n" + "namespace = ns\n" "path = p\n" "pch_external_checksum = true\n" "prefix_command = pc\n" @@ -461,6 +462,7 @@ TEST_CASE("Config::visit_items") "(test.conf) log_file = lf", "(test.conf) max_files = 4711", "(test.conf) max_size = 98.7M", + "(test.conf) namespace = ns", "(test.conf) path = p", "(test.conf) pch_external_checksum = true", "(test.conf) prefix_command = pc",