From: Joel Rosdahl Date: Tue, 27 Sep 2022 18:54:29 +0000 (+0200) Subject: feat: Implement "remote only" mode X-Git-Tag: v4.7~40 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d5782080d6f12a6b246cd24493c76cf07f626413;p=thirdparty%2Fccache.git feat: Implement "remote only" mode Closes #1010. --- diff --git a/doc/MANUAL.adoc b/doc/MANUAL.adoc index 0e59898fe..8577ac1f2 100644 --- a/doc/MANUAL.adoc +++ b/doc/MANUAL.adoc @@ -881,12 +881,20 @@ temporary files otherwise. You may also want to set <> to If true, ccache will not use any previously stored result. New results will still be cached, possibly overwriting any pre-existing results. +[#config_remote_only] +*remote_only* (*CCACHE_REMOTE_ONLY* or *CCACHE_NOREMOTE_ONLY*, see _<>_ above):: + + If true, ccache will only use <>. The + default is false. Note that cache statistics counters will still be kept in + the local cache directory unless <> is false. + [#config_remote_storage] *remote_storage* (*CCACHE_REMOTE_STORAGE*):: This option specifies one or several storage backends (separated by space) - to query after checking the local cache. See _<>_ - for documentation of syntax and available backends. + to query after checking the local cache (unless + <> is true). See _<>_ for documentation of syntax and available backends. + Examples: + @@ -1034,10 +1042,12 @@ filesystem as the `CCACHE_DIR` path, but this requirement has been relaxed. == Remote storage backends The <> option lets you configure ccache -to use one or several other storage backends in addition to the local cache -storage located in <>. Note that cache statistics -counters will still be kept in the local cache directory -- remote storage -backends only store compilation results and manifests. +to use one or several remote storage backends. By default, the local cache +directory located in <> will be queried first and +remote storage second, but <> can be set to +true to disable local storage. Note that cache statistics counters will still be +kept in the local cache directory -- remote storage backends only store +compilation results and manifests. A remote storage backend is specified with a URL, optionally followed by a pipe (`|`) and a pipe-separated list of attributes. An attribute is _key_=_value_ or @@ -1078,7 +1088,8 @@ Examples: === Storage interaction The table below describes the interaction between local and remote storage on -cache hits and misses: +cache hits and misses if <> is false (which is +the default): [options="header",cols="20%,20%,60%"] |============================================================================== @@ -1094,6 +1105,16 @@ cache hits and misses: ^[2]^ Unless local storage is set to share its cache hits with the <> option. +If <> is true: + +[options="header",cols="20%,20%,60%"] +|============================================================================== +| *Local storage* | *Remote storage* | *What happens* + +| - | miss | Compile, write to remote, don't write to local +| - | hit | Read from remote, don't write to local + +|============================================================================== === File storage backend diff --git a/src/Config.cpp b/src/Config.cpp index de5b52470..9d2b3ae44 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -91,6 +91,7 @@ enum class ConfigItem { read_only, read_only_direct, recache, + remote_only, remote_storage, reshare, run_second_cpp, @@ -145,6 +146,7 @@ const std::unordered_map k_config_key_table = {"read_only", {ConfigItem::read_only}}, {"read_only_direct", {ConfigItem::read_only_direct}}, {"recache", {ConfigItem::recache}}, + {"remote_only", {ConfigItem::remote_only}}, {"remote_storage", {ConfigItem::remote_storage}}, {"reshare", {ConfigItem::reshare}}, {"run_second_cpp", {ConfigItem::run_second_cpp}}, @@ -193,6 +195,7 @@ const std::unordered_map k_env_variable_table = { {"READONLY", "read_only"}, {"READONLY_DIRECT", "read_only_direct"}, {"RECACHE", "recache"}, + {"REMOTE_ONLY", "remote_only"}, {"REMOTE_STORAGE", "remote_storage"}, {"RESHARE", "reshare"}, {"SECONDARY_STORAGE", "remote_storage"}, // Alias for CCACHE_REMOTE_STORAGE @@ -775,6 +778,9 @@ Config::get_string_value(const std::string& key) const case ConfigItem::recache: return format_bool(m_recache); + case ConfigItem::remote_only: + return format_bool(m_remote_only); + case ConfigItem::remote_storage: return m_remote_storage; @@ -1028,6 +1034,10 @@ Config::set_item(const std::string& key, m_recache = parse_bool(value, env_var_key, negate); break; + case ConfigItem::remote_only: + m_remote_only = parse_bool(value, env_var_key, negate); + break; + case ConfigItem::remote_storage: m_remote_storage = Util::expand_environment_variables(value); break; diff --git a/src/Config.hpp b/src/Config.hpp index 7ee59921f..7a01b35e4 100644 --- a/src/Config.hpp +++ b/src/Config.hpp @@ -84,6 +84,7 @@ public: bool read_only() const; bool read_only_direct() const; bool recache() const; + bool remote_only() const; const std::string& remote_storage() const; bool reshare() const; bool run_second_cpp() const; @@ -193,6 +194,7 @@ private: bool m_recache = false; bool m_reshare = false; bool m_run_second_cpp = true; + bool m_remote_only = false; std::string m_remote_storage; core::Sloppiness m_sloppiness; bool m_stats = true; @@ -437,6 +439,12 @@ Config::run_second_cpp() const return m_run_second_cpp; } +inline bool +Config::remote_only() const +{ + return m_remote_only; +} + inline const std::string& Config::remote_storage() const { diff --git a/src/ccache.cpp b/src/ccache.cpp index 6cf1c53ab..26b4d9fd7 100644 --- a/src/ccache.cpp +++ b/src/ccache.cpp @@ -901,10 +901,14 @@ write_result(Context& ctx, core::CacheEntry::Header header(ctx.config, core::CacheEntryType::result); const auto cache_entry_data = core::CacheEntry::serialize(header, serializer); - const auto raw_files = serializer.get_raw_files(); - if (!raw_files.empty()) { - ctx.storage.local.put_raw_files(result_key, raw_files); + + if (!ctx.config.remote_only()) { + const auto& raw_files = serializer.get_raw_files(); + if (!raw_files.empty()) { + ctx.storage.local.put_raw_files(result_key, raw_files); + } } + ctx.storage.put(result_key, core::CacheEntryType::result, cache_entry_data); return true; @@ -1742,7 +1746,7 @@ hash_direct_mode_specific_data(Context& ctx, } }); MTR_END("manifest", "manifest_get"); - if (read_manifests > 1) { + if (read_manifests > 1 && !ctx.config.remote_only()) { MTR_SCOPE("manifest", "merge"); LOG("Storing merged manifest {} locally", manifest_key->to_string()); core::CacheEntry::Header header(ctx.config, core::CacheEntryType::manifest); diff --git a/src/storage/Storage.cpp b/src/storage/Storage.cpp index 11490edcc..a607be50a 100644 --- a/src/storage/Storage.cpp +++ b/src/storage/Storage.cpp @@ -237,20 +237,24 @@ Storage::get(const Digest& key, { MTR_SCOPE("storage", "get"); - auto value = local.get(key, type); - local.increment_statistic(value ? core::Statistic::local_storage_hit - : core::Statistic::local_storage_miss); - if (value) { - if (m_config.reshare()) { - put_in_remote_storage(key, *value, true); - } - if (entry_receiver(std::move(*value))) { - return; + if (!m_config.remote_only()) { + auto value = local.get(key, type); + local.increment_statistic(value ? core::Statistic::local_storage_hit + : core::Statistic::local_storage_miss); + if (value) { + if (m_config.reshare()) { + put_in_remote_storage(key, *value, true); + } + if (entry_receiver(std::move(*value))) { + return; + } } } get_from_remote_storage(key, [&](util::Bytes&& data) { - local.put(key, type, data, true); + if (!m_config.remote_only()) { + local.put(key, type, data, true); + } return entry_receiver(std::move(data)); }); } @@ -262,7 +266,9 @@ Storage::put(const Digest& key, { MTR_SCOPE("storage", "put"); - local.put(key, type, value); + if (!m_config.remote_only()) { + local.put(key, type, value); + } put_in_remote_storage(key, value, false); } @@ -271,7 +277,9 @@ Storage::remove(const Digest& key, const core::CacheEntryType type) { MTR_SCOPE("storage", "remove"); - local.remove(key, type); + if (!m_config.remote_only()) { + local.remove(key, type); + } remove_from_remote_storage(key); } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 4cd38a38a..939a220c9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -54,6 +54,7 @@ addtest(readonly) addtest(readonly_direct) addtest(remote_file) addtest(remote_http) +addtest(remote_only) addtest(remote_redis) addtest(remote_redis_unix) addtest(remote_url) diff --git a/test/suites/remote_only.bash b/test/suites/remote_only.bash new file mode 100644 index 000000000..3dea32ac3 --- /dev/null +++ b/test/suites/remote_only.bash @@ -0,0 +1,33 @@ +SUITE_remote_only_SETUP() { + unset CCACHE_NODIRECT + export CCACHE_REMOTE_STORAGE="file:$PWD/remote" + export CCACHE_REMOTE_ONLY=1 + + generate_code 1 test.c +} + +SUITE_remote_only() { + # ------------------------------------------------------------------------- + TEST "Base case" + + $CCACHE_COMPILE -c test.c + expect_stat direct_cache_hit 0 + expect_stat cache_miss 1 + expect_stat files_in_cache 0 + expect_stat local_storage_hit 0 + expect_stat local_storage_miss 0 + expect_stat remote_storage_hit 0 + expect_stat remote_storage_miss 2 # result + manifest + expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest + + $CCACHE_COMPILE -c test.c + expect_stat direct_cache_hit 1 + expect_stat cache_miss 1 + expect_stat files_in_cache 0 + expect_stat local_storage_hit 0 + expect_stat local_storage_miss 0 + expect_stat remote_storage_hit 2 + expect_stat remote_storage_miss 2 + expect_stat files_in_cache 0 + expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest +} diff --git a/unittest/test_Config.cpp b/unittest/test_Config.cpp index 144568114..0226101f0 100644 --- a/unittest/test_Config.cpp +++ b/unittest/test_Config.cpp @@ -71,6 +71,7 @@ TEST_CASE("Config: default values") CHECK_FALSE(config.read_only()); CHECK_FALSE(config.read_only_direct()); CHECK_FALSE(config.recache()); + CHECK_FALSE(config.remote_only()); CHECK(config.remote_storage().empty()); CHECK_FALSE(config.reshare()); CHECK(config.run_second_cpp()); @@ -413,6 +414,7 @@ TEST_CASE("Config::visit_items") "read_only = true\n" "read_only_direct = true\n" "recache = true\n" + "remote_only = true\n" "remote_storage = rs\n" "reshare = true\n" "run_second_cpp = false\n" @@ -473,6 +475,7 @@ TEST_CASE("Config::visit_items") "(test.conf) read_only = true", "(test.conf) read_only_direct = true", "(test.conf) recache = true", + "(test.conf) remote_only = true", "(test.conf) remote_storage = rs", "(test.conf) reshare = true", "(test.conf) run_second_cpp = false",