From: Joel Rosdahl Date: Mon, 9 Aug 2021 18:55:11 +0000 (+0200) Subject: feat: Add reshare config option X-Git-Tag: v4.4~33 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=9af44de39e7708af4e73ed8e4f6212f96f2c4365;p=thirdparty%2Fccache.git feat: Add reshare config option Setting reshare (CCACHE_RESHARE) to true make ccache send results to secondary storage even for primary storage cache hits. The default is false. --- diff --git a/doc/MANUAL.adoc b/doc/MANUAL.adoc index 9a5b0b352..6b68f8f36 100644 --- a/doc/MANUAL.adoc +++ b/doc/MANUAL.adoc @@ -785,6 +785,12 @@ temporary files otherwise. You may also want to set <> to If true, ccache will not use any previously stored result. New results will still be cached, possibly overwriting any pre-existing results. +[#config_reshare] +*reshare* (*CCACHE_RESHARE* or *CCACHE_NORESHARE*, see _<>_ above):: + + If true, ccache will write results to secondary storage even for primary + storage cache hits. The default is false. + [#config_run_second_cpp] *run_second_cpp* (*CCACHE_CPP2* or *CCACHE_NOCPP2*, see _<>_ above):: diff --git a/src/Config.cpp b/src/Config.cpp index 723588085..0ce230313 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -93,6 +93,7 @@ enum class ConfigItem { read_only, read_only_direct, recache, + reshare, run_second_cpp, secondary_storage, sloppiness, @@ -136,6 +137,7 @@ const std::unordered_map k_config_key_table = { {"read_only", ConfigItem::read_only}, {"read_only_direct", ConfigItem::read_only_direct}, {"recache", ConfigItem::recache}, + {"reshare", ConfigItem::reshare}, {"run_second_cpp", ConfigItem::run_second_cpp}, {"secondary_storage", ConfigItem::secondary_storage}, {"sloppiness", ConfigItem::sloppiness}, @@ -181,6 +183,7 @@ const std::unordered_map k_env_variable_table = { {"READONLY", "read_only"}, {"READONLY_DIRECT", "read_only_direct"}, {"RECACHE", "recache"}, + {"RESHARE", "reshare"}, {"SECONDARY_STORAGE", "secondary_storage"}, {"SLOPPINESS", "sloppiness"}, {"STATS", "stats"}, @@ -709,6 +712,9 @@ Config::get_string_value(const std::string& key) const case ConfigItem::recache: return format_bool(m_recache); + case ConfigItem::reshare: + return format_bool(m_reshare); + case ConfigItem::run_second_cpp: return format_bool(m_run_second_cpp); @@ -953,6 +959,10 @@ Config::set_item(const std::string& key, m_recache = parse_bool(value, env_var_key, negate); break; + case ConfigItem::reshare: + m_reshare = parse_bool(value, env_var_key, negate); + break; + case ConfigItem::run_second_cpp: m_run_second_cpp = parse_bool(value, env_var_key, negate); break; diff --git a/src/Config.hpp b/src/Config.hpp index f12b65b58..acb09e990 100644 --- a/src/Config.hpp +++ b/src/Config.hpp @@ -75,6 +75,7 @@ public: bool read_only() const; bool read_only_direct() const; bool recache() const; + bool reshare() const; bool run_second_cpp() const; const std::string& secondary_storage() const; core::Sloppiness sloppiness() const; @@ -169,6 +170,7 @@ private: bool m_read_only = false; bool m_read_only_direct = false; bool m_recache = false; + bool m_reshare = false; bool m_run_second_cpp = true; std::string m_secondary_storage; core::Sloppiness m_sloppiness; @@ -388,6 +390,12 @@ Config::recache() const return m_recache; } +inline bool +Config::reshare() const +{ + return m_reshare; +} + inline bool Config::run_second_cpp() const { diff --git a/src/storage/Storage.cpp b/src/storage/Storage.cpp index 828e32370..d1f0d5c4d 100644 --- a/src/storage/Storage.cpp +++ b/src/storage/Storage.cpp @@ -227,13 +227,31 @@ nonstd::optional Storage::get(const Digest& key, const core::CacheEntryType type) { const auto path = primary.get(key, type); + primary.increment_statistic(path ? core::Statistic::primary_storage_hit + : core::Statistic::primary_storage_miss); if (path) { - primary.increment_statistic(core::Statistic::primary_storage_hit); + if (m_config.reshare()) { + // Temporary optimization until primary storage API has been refactored to + // pass data via memory instead of files. + const bool should_put_in_secondary_storage = + std::any_of(m_secondary_storages.begin(), + m_secondary_storages.end(), + [](const auto& entry) { return !entry->config.read_only; }); + if (should_put_in_secondary_storage) { + std::string value; + try { + value = Util::read_file(*path); + } catch (const core::Error& e) { + LOG("Failed to read {}: {}", *path, e.what()); + return path; // Don't indicate failure since primary storage was OK. + } + put_in_secondary_storage(key, value, true); + } + } + return path; } - primary.increment_statistic(core::Statistic::primary_storage_miss); - const auto value = get_from_secondary_storage(key); if (!value) { return nonstd::nullopt; @@ -284,7 +302,7 @@ Storage::put(const Digest& key, LOG("Failed to read {}: {}", *path, e.what()); return true; // Don't indicate failure since primary storage was OK. } - put_in_secondary_storage(key, value); + put_in_secondary_storage(key, value, false); } return true; @@ -462,7 +480,9 @@ Storage::get_from_secondary_storage(const Digest& key) } void -Storage::put_in_secondary_storage(const Digest& key, const std::string& value) +Storage::put_in_secondary_storage(const Digest& key, + const std::string& value, + bool only_if_missing) { for (const auto& entry : m_secondary_storages) { auto backend = get_backend(*entry, key, "putting in", true); @@ -471,7 +491,7 @@ Storage::put_in_secondary_storage(const Digest& key, const std::string& value) } Timer timer; - const auto result = backend->impl->put(key, value); + const auto result = backend->impl->put(key, value, only_if_missing); const auto ms = timer.measure_ms(); if (!result) { // The backend is expected to log details about the error. @@ -481,7 +501,7 @@ Storage::put_in_secondary_storage(const Digest& key, const std::string& value) const bool stored = *result; LOG("{} {} in {} ({:.2f} ms)", - stored ? "Stored" : "Failed to store", + stored ? "Stored" : "Did not have to store", key.to_string(), entry->url_for_logging, ms); diff --git a/src/storage/Storage.hpp b/src/storage/Storage.hpp index 9238d5724..310eebfd3 100644 --- a/src/storage/Storage.hpp +++ b/src/storage/Storage.hpp @@ -80,7 +80,9 @@ private: const bool for_writing); nonstd::optional get_from_secondary_storage(const Digest& key); - void put_in_secondary_storage(const Digest& key, const std::string& value); + void put_in_secondary_storage(const Digest& key, + const std::string& value, + bool only_if_missing); void remove_from_secondary_storage(const Digest& key); }; diff --git a/test/suites/secondary_file.bash b/test/suites/secondary_file.bash index a4de8f384..85ae6285b 100644 --- a/test/suites/secondary_file.bash +++ b/test/suites/secondary_file.bash @@ -1,3 +1,6 @@ +# This test suite verified both the file storage backend and the secondary +# storage framework itself. + SUITE_secondary_file_SETUP() { unset CCACHE_NODIRECT export CCACHE_SECONDARY_STORAGE="file:$PWD/secondary" @@ -174,4 +177,46 @@ SUITE_secondary_file() { if [ ! -d secondary/a ] && [ ! -d secondary/b ]; then test_failed "Expected secondary/a or secondary/b to exist" fi + + # ------------------------------------------------------------------------- + TEST "Reshare" + + CCACHE_SECONDARY_STORAGE="" $CCACHE_COMPILE -c test.c + expect_stat direct_cache_hit 0 + expect_stat cache_miss 1 + expect_stat files_in_cache 2 + expect_stat primary_storage_hit 0 + expect_stat primary_storage_miss 2 + expect_stat secondary_storage_hit 0 + expect_stat secondary_storage_miss 0 + expect_missing secondary + + $CCACHE_COMPILE -c test.c + expect_stat direct_cache_hit 1 + expect_stat cache_miss 1 + expect_stat primary_storage_hit 2 + expect_stat primary_storage_miss 2 + expect_stat secondary_storage_hit 0 + expect_stat secondary_storage_miss 0 + expect_missing secondary + + CCACHE_RESHARE=1 $CCACHE_COMPILE -c test.c + expect_stat direct_cache_hit 2 + expect_stat cache_miss 1 + expect_stat primary_storage_hit 4 + expect_stat primary_storage_miss 2 + expect_stat secondary_storage_hit 0 + expect_stat secondary_storage_miss 0 + expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest + + $CCACHE -C >/dev/null + + $CCACHE_COMPILE -c test.c + expect_stat direct_cache_hit 3 + expect_stat cache_miss 1 + expect_stat primary_storage_hit 4 + expect_stat primary_storage_miss 4 + expect_stat secondary_storage_hit 2 + expect_stat secondary_storage_miss 0 + expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest } diff --git a/unittest/test_Config.cpp b/unittest/test_Config.cpp index 16b6942cd..3b62174c4 100644 --- a/unittest/test_Config.cpp +++ b/unittest/test_Config.cpp @@ -70,6 +70,7 @@ TEST_CASE("Config: default values") CHECK_FALSE(config.read_only()); CHECK_FALSE(config.read_only_direct()); CHECK_FALSE(config.recache()); + CHECK_FALSE(config.reshare()); CHECK(config.run_second_cpp()); CHECK(config.sloppiness().to_bitmask() == 0); CHECK(config.stats()); @@ -125,6 +126,7 @@ TEST_CASE("Config::update_from_file") "read_only = true\n" "read_only_direct = true\n" "recache = true\n" + "reshare = true\n" "run_second_cpp = false\n" "sloppiness = time_macros ,include_file_mtime" " include_file_ctime,file_stat_matches,file_stat_matches_ctime,pch_defines" @@ -164,6 +166,7 @@ TEST_CASE("Config::update_from_file") CHECK(config.read_only()); CHECK(config.read_only_direct()); CHECK(config.recache()); + CHECK(config.reshare()); CHECK_FALSE(config.run_second_cpp()); CHECK(config.sloppiness().to_bitmask() == (static_cast(core::Sloppy::include_file_mtime) @@ -406,6 +409,7 @@ TEST_CASE("Config::visit_items") "read_only = true\n" "read_only_direct = true\n" "recache = true\n" + "reshare = true\n" "run_second_cpp = false\n" "secondary_storage = ss\n" "sloppiness = include_file_mtime, include_file_ctime, time_macros," @@ -464,6 +468,7 @@ TEST_CASE("Config::visit_items") "(test.conf) read_only = true", "(test.conf) read_only_direct = true", "(test.conf) recache = true", + "(test.conf) reshare = true", "(test.conf) run_second_cpp = false", "(test.conf) secondary_storage = ss", "(test.conf) sloppiness = include_file_mtime, include_file_ctime,"