feat: Add statistics for primary/secondary hits/miss

diff --git a/src/core/Statistic.hpp b/src/core/Statistic.hpp
index 43cc94f166cb3c990b9b0c32471045b5f4ec1604..8b67e96a80e6eddd9e3172122ea80e35e08677c0 100644 (file)
--- a/src/core/Statistic.hpp
+++ b/src/core/Statistic.hpp
@@ -57,6 +57,12 @@ enum class Statistic {
   could_not_use_modules = 32,
   direct_cache_miss = 33,
   preprocessed_cache_miss = 34,
+  primary_storage_hit = 35,
+  primary_storage_miss = 36,
+  secondary_storage_hit = 37,
+  secondary_storage_miss = 38,
+  secondary_storage_error = 39,
+  secondary_storage_timeout = 40,
 
   END
 };

diff --git a/src/core/Statistics.cpp b/src/core/Statistics.cpp
index ce3a24b18d503a62f742736b4db0a12d984a0ade..9351f186eddd1fe54cf849401fd7623eff082763 100644 (file)
--- a/src/core/Statistics.cpp
+++ b/src/core/Statistics.cpp
@@ -81,6 +81,12 @@ const StatisticsField k_statistics_fields[] = {
   STATISTICS_FIELD(cache_miss, "cache miss", FLAG_ALWAYS),
   STATISTICS_FIELD(direct_cache_miss, "cache miss (direct)"),
   STATISTICS_FIELD(preprocessed_cache_miss, "cache miss (preprocessed)"),
+  STATISTICS_FIELD(primary_storage_hit, "primary storage hit"),
+  STATISTICS_FIELD(primary_storage_miss, "primary storage miss"),
+  STATISTICS_FIELD(secondary_storage_hit, "secondary storage hit"),
+  STATISTICS_FIELD(secondary_storage_miss, "secondary storage miss"),
+  STATISTICS_FIELD(secondary_storage_error, "secondary storage error"),
+  STATISTICS_FIELD(secondary_storage_timeout, "secondary storage timeout"),
   STATISTICS_FIELD(called_for_link, "called for link"),
   STATISTICS_FIELD(called_for_preprocessing, "called for preprocessing"),
   STATISTICS_FIELD(multiple_source_files, "multiple source files"),

diff --git a/src/storage/Storage.cpp b/src/storage/Storage.cpp
index 8d531a3eac5432262105e4eb459c0a4c68097ec3..828e323700c7ddd168edaf8e8d08bf3452d55dbc 100644 (file)
--- a/src/storage/Storage.cpp
+++ b/src/storage/Storage.cpp
@@ -24,6 +24,7 @@
 #include <TemporaryFile.hpp>
 #include <Util.hpp>
 #include <assertions.hpp>
+#include <core/Statistic.hpp>
 #include <core/exceptions.hpp>
 #include <fmtmacros.hpp>
 #include <storage/secondary/FileStorage.hpp>
@@ -31,7 +32,6 @@
 #ifdef HAVE_REDIS_STORAGE_BACKEND
 #  include <storage/secondary/RedisStorage.hpp>
 #endif
-#include <storage/secondary/SecondaryStorage.hpp>
 #include <util/Timer.hpp>
 #include <util/Tokenizer.hpp>
 #include <util/expected.hpp>
@@ -228,9 +228,12 @@ Storage::get(const Digest& key, const core::CacheEntryType type)
 {
   const auto path = primary.get(key, type);
   if (path) {
+    primary.increment_statistic(core::Statistic::primary_storage_hit);
     return path;
   }
 
+  primary.increment_statistic(core::Statistic::primary_storage_miss);
+
   const auto value = get_from_secondary_storage(key);
   if (!value) {
     return nonstd::nullopt;
@@ -324,14 +327,17 @@ Storage::add_secondary_storages()
   }
 }
 
-static void
-mark_backend_as_failed(
+void
+Storage::mark_backend_as_failed(
   SecondaryStorageBackendEntry& backend_entry,
   const secondary::SecondaryStorage::Backend::Failure failure)
 {
   // The backend is expected to log details about the error.
   backend_entry.failed = true;
-  (void)failure; // TODO: Update statistics.
+  primary.increment_statistic(
+    failure == secondary::SecondaryStorage::Backend::Failure::timeout
+      ? core::Statistic::secondary_storage_timeout
+      : core::Statistic::secondary_storage_error);
 }
 
 static double
@@ -370,11 +376,11 @@ get_shard_url(const Digest& key,
   return util::replace_first(url, "*", best_shard);
 }
 
-static SecondaryStorageBackendEntry*
-get_backend(SecondaryStorageEntry& entry,
-            const Digest& key,
-            nonstd::string_view operation_description,
-            const bool for_writing)
+SecondaryStorageBackendEntry*
+Storage::get_backend(SecondaryStorageEntry& entry,
+                     const Digest& key,
+                     const nonstd::string_view operation_description,
+                     const bool for_writing)
 {
   if (for_writing && entry.config.read_only) {
     LOG("Not {} {} since it is read-only",
@@ -441,12 +447,14 @@ Storage::get_from_secondary_storage(const Digest& key)
           key.to_string(),
           backend->url_for_logging,
           ms);
+      primary.increment_statistic(core::Statistic::secondary_storage_hit);
       return *value;
     } else {
       LOG("No {} in {} ({:.2f} ms)",
           key.to_string(),
           backend->url_for_logging,
           ms);
+      primary.increment_statistic(core::Statistic::secondary_storage_miss);
     }
   }
 

diff --git a/src/storage/Storage.hpp b/src/storage/Storage.hpp
index fe905fd39af52cc1e03d53c785a121e6e3b00327..9238d5724a49fed4a1c560ecbea8d42b1512cfcc 100644 (file)
--- a/src/storage/Storage.hpp
+++ b/src/storage/Storage.hpp
@@ -20,6 +20,7 @@
 
 #include <core/types.hpp>
 #include <storage/primary/PrimaryStorage.hpp>
+#include <storage/secondary/SecondaryStorage.hpp>
 #include <storage/types.hpp>
 
 #include <third_party/nonstd/optional.hpp>
@@ -35,6 +36,7 @@ namespace storage {
 
 std::string get_features();
 
+struct SecondaryStorageBackendEntry;
 struct SecondaryStorageEntry;
 
 class Storage
@@ -66,8 +68,20 @@ private:
   std::vector<std::string> m_tmp_files;
 
   void add_secondary_storages();
+
+  void
+  mark_backend_as_failed(SecondaryStorageBackendEntry& backend_entry,
+                         secondary::SecondaryStorage::Backend::Failure failure);
+
+  SecondaryStorageBackendEntry*
+  get_backend(SecondaryStorageEntry& entry,
+              const Digest& key,
+              nonstd::string_view operation_description,
+              const bool for_writing);
   nonstd::optional<std::string> get_from_secondary_storage(const Digest& key);
+
   void put_in_secondary_storage(const Digest& key, const std::string& value);
+
   void remove_from_secondary_storage(const Digest& key);
 };
 

diff --git a/test/suites/base.bash b/test/suites/base.bash
index 84f51a48c380cc304c64b190ec5d1f0305eebe00..3996deaf52fb8cc7747d6580db43833828b8f163 100644 (file)
--- a/test/suites/base.bash
+++ b/test/suites/base.bash
@@ -9,6 +9,10 @@ base_tests() {
     expect_stat cache_miss 1
     expect_stat direct_cache_miss 0
     expect_stat preprocessed_cache_miss 1
+    expect_stat primary_storage_hit 0
+    expect_stat primary_storage_miss 1
+    expect_stat secondary_storage_hit 0
+    expect_stat secondary_storage_miss 0
     expect_stat files_in_cache 1
     expect_equal_object_files reference_test1.o test1.o
 
@@ -17,6 +21,10 @@ base_tests() {
     expect_stat cache_miss 1
     expect_stat direct_cache_miss 0
     expect_stat preprocessed_cache_miss 1
+    expect_stat primary_storage_hit 1
+    expect_stat primary_storage_miss 1
+    expect_stat secondary_storage_hit 0
+    expect_stat secondary_storage_miss 0
     expect_stat files_in_cache 1
     expect_equal_object_files reference_test1.o test1.o
 

diff --git a/test/suites/direct.bash b/test/suites/direct.bash
index 660ee07b654469d7022df3544a51fc4a07179624..4f26a63eb2242729c5743da0f7ada7807400cb2f 100644 (file)
--- a/test/suites/direct.bash
+++ b/test/suites/direct.bash
@@ -35,6 +35,10 @@ SUITE_direct() {
     expect_stat cache_miss 1
     expect_stat direct_cache_miss 1
     expect_stat preprocessed_cache_miss 1
+    expect_stat primary_storage_hit 0
+    expect_stat primary_storage_miss 2 # result + manifest
+    expect_stat secondary_storage_hit 0
+    expect_stat secondary_storage_miss 0
     expect_stat files_in_cache 2 # result + manifest
     expect_equal_object_files reference_test.o test.o
 
@@ -47,6 +51,10 @@ SUITE_direct() {
     expect_stat cache_miss 1
     expect_stat direct_cache_miss 1
     expect_stat preprocessed_cache_miss 1
+    expect_stat primary_storage_hit 2 # result + manifest
+    expect_stat primary_storage_miss 2 # result + manifest
+    expect_stat secondary_storage_hit 0
+    expect_stat secondary_storage_miss 0
     expect_stat files_in_cache 2
     expect_equal_object_files reference_test.o test.o
     expect_newer_than $manifest_file test.c

diff --git a/test/suites/secondary_file.bash b/test/suites/secondary_file.bash
index 0ddebb320e49d06db53337ada8f3fd6b1a20873e..a4de8f3842e1502966d9298c5ae3ae37f27d2519 100644 (file)
--- a/test/suites/secondary_file.bash
+++ b/test/suites/secondary_file.bash
@@ -7,12 +7,16 @@ SUITE_secondary_file_SETUP() {
 
 SUITE_secondary_file() {
     # -------------------------------------------------------------------------
-    TEST "Subdirs layout"
+    TEST "Base case"
 
     $CCACHE_COMPILE -c test.c
     expect_stat direct_cache_hit 0
     expect_stat cache_miss 1
     expect_stat files_in_cache 2
+    expect_stat primary_storage_hit 0
+    expect_stat primary_storage_miss 2 # result + manifest
+    expect_stat secondary_storage_hit 0
+    expect_stat secondary_storage_miss 2 # result + manifest
     expect_exists secondary/CACHEDIR.TAG
     subdirs=$(find secondary -type d | wc -l)
     if [ "${subdirs}" -lt 2 ]; then # "secondary" itself counts as one
@@ -23,6 +27,10 @@ SUITE_secondary_file() {
     $CCACHE_COMPILE -c test.c
     expect_stat direct_cache_hit 1
     expect_stat cache_miss 1
+    expect_stat primary_storage_hit 2 # result + manifest
+    expect_stat primary_storage_miss 2 # result + manifest
+    expect_stat secondary_storage_hit 0
+    expect_stat secondary_storage_miss 2 # result + manifest
     expect_stat files_in_cache 2
     expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
 
@@ -33,6 +41,10 @@ SUITE_secondary_file() {
     $CCACHE_COMPILE -c test.c
     expect_stat direct_cache_hit 2
     expect_stat cache_miss 1
+    expect_stat primary_storage_hit 2
+    expect_stat primary_storage_miss 4 # 2 * (result + manifest)
+    expect_stat secondary_storage_hit 2 # result + manifest
+    expect_stat secondary_storage_miss 2 # result + manifest
     expect_stat files_in_cache 2 # fetched from secondary
     expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
 

diff --git a/test/suites/stats_log.bash b/test/suites/stats_log.bash
index a55de6e06d8c67f4a78e2b0f6137c9da8895af2f..e90efaf5479fdc90f9bbce5849c90f69df712645 100644 (file)
--- a/test/suites/stats_log.bash
+++ b/test/suites/stats_log.bash
@@ -20,6 +20,8 @@ SUITE_stats_log() {
 cache_miss
 direct_cache_miss
 preprocessed_cache_miss
+primary_storage_miss
 # test.c
-direct_cache_hit"
+direct_cache_hit
+primary_storage_hit"
 }