From 1f6144f87ba75d7f66625e71eb4e0070f6b23bb2 Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Sun, 8 Aug 2021 19:59:58 +0200 Subject: [PATCH] feat: Add statistics for primary/secondary hits/miss --- src/core/Statistic.hpp | 6 ++++++ src/core/Statistics.cpp | 6 ++++++ src/storage/Storage.cpp | 26 +++++++++++++++++--------- src/storage/Storage.hpp | 14 ++++++++++++++ test/suites/base.bash | 8 ++++++++ test/suites/direct.bash | 8 ++++++++ test/suites/secondary_file.bash | 14 +++++++++++++- test/suites/stats_log.bash | 4 +++- 8 files changed, 75 insertions(+), 11 deletions(-) diff --git a/src/core/Statistic.hpp b/src/core/Statistic.hpp index 43cc94f16..8b67e96a8 100644 --- a/src/core/Statistic.hpp +++ b/src/core/Statistic.hpp @@ -57,6 +57,12 @@ enum class Statistic { could_not_use_modules = 32, direct_cache_miss = 33, preprocessed_cache_miss = 34, + primary_storage_hit = 35, + primary_storage_miss = 36, + secondary_storage_hit = 37, + secondary_storage_miss = 38, + secondary_storage_error = 39, + secondary_storage_timeout = 40, END }; diff --git a/src/core/Statistics.cpp b/src/core/Statistics.cpp index ce3a24b18..9351f186e 100644 --- a/src/core/Statistics.cpp +++ b/src/core/Statistics.cpp @@ -81,6 +81,12 @@ const StatisticsField k_statistics_fields[] = { STATISTICS_FIELD(cache_miss, "cache miss", FLAG_ALWAYS), STATISTICS_FIELD(direct_cache_miss, "cache miss (direct)"), STATISTICS_FIELD(preprocessed_cache_miss, "cache miss (preprocessed)"), + STATISTICS_FIELD(primary_storage_hit, "primary storage hit"), + STATISTICS_FIELD(primary_storage_miss, "primary storage miss"), + STATISTICS_FIELD(secondary_storage_hit, "secondary storage hit"), + STATISTICS_FIELD(secondary_storage_miss, "secondary storage miss"), + STATISTICS_FIELD(secondary_storage_error, "secondary storage error"), + STATISTICS_FIELD(secondary_storage_timeout, "secondary storage timeout"), STATISTICS_FIELD(called_for_link, "called for link"), STATISTICS_FIELD(called_for_preprocessing, "called for preprocessing"), STATISTICS_FIELD(multiple_source_files, "multiple source files"), diff --git a/src/storage/Storage.cpp b/src/storage/Storage.cpp index 8d531a3ea..828e32370 100644 --- a/src/storage/Storage.cpp +++ b/src/storage/Storage.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -31,7 +32,6 @@ #ifdef HAVE_REDIS_STORAGE_BACKEND # include #endif -#include #include #include #include @@ -228,9 +228,12 @@ Storage::get(const Digest& key, const core::CacheEntryType type) { const auto path = primary.get(key, type); if (path) { + primary.increment_statistic(core::Statistic::primary_storage_hit); return path; } + primary.increment_statistic(core::Statistic::primary_storage_miss); + const auto value = get_from_secondary_storage(key); if (!value) { return nonstd::nullopt; @@ -324,14 +327,17 @@ Storage::add_secondary_storages() } } -static void -mark_backend_as_failed( +void +Storage::mark_backend_as_failed( SecondaryStorageBackendEntry& backend_entry, const secondary::SecondaryStorage::Backend::Failure failure) { // The backend is expected to log details about the error. backend_entry.failed = true; - (void)failure; // TODO: Update statistics. + primary.increment_statistic( + failure == secondary::SecondaryStorage::Backend::Failure::timeout + ? core::Statistic::secondary_storage_timeout + : core::Statistic::secondary_storage_error); } static double @@ -370,11 +376,11 @@ get_shard_url(const Digest& key, return util::replace_first(url, "*", best_shard); } -static SecondaryStorageBackendEntry* -get_backend(SecondaryStorageEntry& entry, - const Digest& key, - nonstd::string_view operation_description, - const bool for_writing) +SecondaryStorageBackendEntry* +Storage::get_backend(SecondaryStorageEntry& entry, + const Digest& key, + const nonstd::string_view operation_description, + const bool for_writing) { if (for_writing && entry.config.read_only) { LOG("Not {} {} since it is read-only", @@ -441,12 +447,14 @@ Storage::get_from_secondary_storage(const Digest& key) key.to_string(), backend->url_for_logging, ms); + primary.increment_statistic(core::Statistic::secondary_storage_hit); return *value; } else { LOG("No {} in {} ({:.2f} ms)", key.to_string(), backend->url_for_logging, ms); + primary.increment_statistic(core::Statistic::secondary_storage_miss); } } diff --git a/src/storage/Storage.hpp b/src/storage/Storage.hpp index fe905fd39..9238d5724 100644 --- a/src/storage/Storage.hpp +++ b/src/storage/Storage.hpp @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -35,6 +36,7 @@ namespace storage { std::string get_features(); +struct SecondaryStorageBackendEntry; struct SecondaryStorageEntry; class Storage @@ -66,8 +68,20 @@ private: std::vector m_tmp_files; void add_secondary_storages(); + + void + mark_backend_as_failed(SecondaryStorageBackendEntry& backend_entry, + secondary::SecondaryStorage::Backend::Failure failure); + + SecondaryStorageBackendEntry* + get_backend(SecondaryStorageEntry& entry, + const Digest& key, + nonstd::string_view operation_description, + const bool for_writing); nonstd::optional get_from_secondary_storage(const Digest& key); + void put_in_secondary_storage(const Digest& key, const std::string& value); + void remove_from_secondary_storage(const Digest& key); }; diff --git a/test/suites/base.bash b/test/suites/base.bash index 84f51a48c..3996deaf5 100644 --- a/test/suites/base.bash +++ b/test/suites/base.bash @@ -9,6 +9,10 @@ base_tests() { expect_stat cache_miss 1 expect_stat direct_cache_miss 0 expect_stat preprocessed_cache_miss 1 + expect_stat primary_storage_hit 0 + expect_stat primary_storage_miss 1 + expect_stat secondary_storage_hit 0 + expect_stat secondary_storage_miss 0 expect_stat files_in_cache 1 expect_equal_object_files reference_test1.o test1.o @@ -17,6 +21,10 @@ base_tests() { expect_stat cache_miss 1 expect_stat direct_cache_miss 0 expect_stat preprocessed_cache_miss 1 + expect_stat primary_storage_hit 1 + expect_stat primary_storage_miss 1 + expect_stat secondary_storage_hit 0 + expect_stat secondary_storage_miss 0 expect_stat files_in_cache 1 expect_equal_object_files reference_test1.o test1.o diff --git a/test/suites/direct.bash b/test/suites/direct.bash index 660ee07b6..4f26a63eb 100644 --- a/test/suites/direct.bash +++ b/test/suites/direct.bash @@ -35,6 +35,10 @@ SUITE_direct() { expect_stat cache_miss 1 expect_stat direct_cache_miss 1 expect_stat preprocessed_cache_miss 1 + expect_stat primary_storage_hit 0 + expect_stat primary_storage_miss 2 # result + manifest + expect_stat secondary_storage_hit 0 + expect_stat secondary_storage_miss 0 expect_stat files_in_cache 2 # result + manifest expect_equal_object_files reference_test.o test.o @@ -47,6 +51,10 @@ SUITE_direct() { expect_stat cache_miss 1 expect_stat direct_cache_miss 1 expect_stat preprocessed_cache_miss 1 + expect_stat primary_storage_hit 2 # result + manifest + expect_stat primary_storage_miss 2 # result + manifest + expect_stat secondary_storage_hit 0 + expect_stat secondary_storage_miss 0 expect_stat files_in_cache 2 expect_equal_object_files reference_test.o test.o expect_newer_than $manifest_file test.c diff --git a/test/suites/secondary_file.bash b/test/suites/secondary_file.bash index 0ddebb320..a4de8f384 100644 --- a/test/suites/secondary_file.bash +++ b/test/suites/secondary_file.bash @@ -7,12 +7,16 @@ SUITE_secondary_file_SETUP() { SUITE_secondary_file() { # ------------------------------------------------------------------------- - TEST "Subdirs layout" + TEST "Base case" $CCACHE_COMPILE -c test.c expect_stat direct_cache_hit 0 expect_stat cache_miss 1 expect_stat files_in_cache 2 + expect_stat primary_storage_hit 0 + expect_stat primary_storage_miss 2 # result + manifest + expect_stat secondary_storage_hit 0 + expect_stat secondary_storage_miss 2 # result + manifest expect_exists secondary/CACHEDIR.TAG subdirs=$(find secondary -type d | wc -l) if [ "${subdirs}" -lt 2 ]; then # "secondary" itself counts as one @@ -23,6 +27,10 @@ SUITE_secondary_file() { $CCACHE_COMPILE -c test.c expect_stat direct_cache_hit 1 expect_stat cache_miss 1 + expect_stat primary_storage_hit 2 # result + manifest + expect_stat primary_storage_miss 2 # result + manifest + expect_stat secondary_storage_hit 0 + expect_stat secondary_storage_miss 2 # result + manifest expect_stat files_in_cache 2 expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest @@ -33,6 +41,10 @@ SUITE_secondary_file() { $CCACHE_COMPILE -c test.c expect_stat direct_cache_hit 2 expect_stat cache_miss 1 + expect_stat primary_storage_hit 2 + expect_stat primary_storage_miss 4 # 2 * (result + manifest) + expect_stat secondary_storage_hit 2 # result + manifest + expect_stat secondary_storage_miss 2 # result + manifest expect_stat files_in_cache 2 # fetched from secondary expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest diff --git a/test/suites/stats_log.bash b/test/suites/stats_log.bash index a55de6e06..e90efaf54 100644 --- a/test/suites/stats_log.bash +++ b/test/suites/stats_log.bash @@ -20,6 +20,8 @@ SUITE_stats_log() { cache_miss direct_cache_miss preprocessed_cache_miss +primary_storage_miss # test.c -direct_cache_hit" +direct_cache_hit +primary_storage_hit" } -- 2.47.2