From: Joel Rosdahl Date: Thu, 22 Jul 2021 11:06:40 +0000 (+0200) Subject: Move/split statistics functionality into core and storage::primary X-Git-Tag: v4.4~100 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=a28af38b6c12eadea1ed833c3fc13177164c9ef1;p=thirdparty%2Fccache.git Move/split statistics functionality into core and storage::primary - The Counters class has been moved to core::StatisticsCounters. - Functionality in the Statistics namespace has been split into: - core::StatsLog: Knows about “stats log” format. - core::Statistics: Knows about properties of statistics fields and how to present them. - storage::primary::PrimaryStorage: Knows how to store and update statistics in the primary storage backend. - storage::primary::StatsFile: Knows about the “stats file” format. --- diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f2c243252..3ace0cb5c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -6,7 +6,6 @@ set( CacheEntryWriter.cpp Config.cpp Context.cpp - Counters.cpp Depfile.cpp Fd.cpp Hash.cpp @@ -20,7 +19,6 @@ set( ResultRetriever.cpp SignalHandler.cpp Stat.cpp - Statistics.cpp TemporaryFile.cpp ThreadPool.cpp Util.cpp diff --git a/src/Context.cpp b/src/Context.cpp index 66bd75cbd..bccf632f8 100644 --- a/src/Context.cpp +++ b/src/Context.cpp @@ -18,7 +18,6 @@ #include "Context.hpp" -#include "Counters.hpp" #include "Logging.hpp" #include "SignalHandler.hpp" #include "Util.hpp" diff --git a/src/Context.hpp b/src/Context.hpp index 96cf134c9..ce40a0870 100644 --- a/src/Context.hpp +++ b/src/Context.hpp @@ -21,7 +21,6 @@ #include "Args.hpp" #include "ArgsInfo.hpp" #include "Config.hpp" -#include "Counters.hpp" #include "Digest.hpp" #include "File.hpp" #include "MiniTrace.hpp" diff --git a/src/Statistics.hpp b/src/Statistics.hpp deleted file mode 100644 index ef7087900..000000000 --- a/src/Statistics.hpp +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright (C) 2020-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "Counters.hpp" - -#include "third_party/nonstd/optional.hpp" - -#include -#include -#include - -class Config; - -namespace Statistics { - -// Read counters from `path`. No lock is acquired. -Counters read(const std::string& path); - -// Read counters from lines of text in `path`. -Counters read_log(const std::string& path); - -// Acquire a lock, read counters from `path`, call `function` with the counters, -// write the counters to `path` and release the lock. Returns the resulting -// counters or nullopt on error (e.g. if the lock could not be acquired). -nonstd::optional update(const std::string& path, - std::function); - -// Write input and result to the file in `path`. -void log_result(const std::string& path, - const std::string& input, - const std::string& result); - -// Return a human-readable string representing the final ccache result, or -// nullopt if there was no result. -nonstd::optional get_result_message(const Counters& counters); - -// Return a machine-readable string representing the final ccache result, or -// nullopt if there was no result. -nonstd::optional get_result_id(const Counters& counters); - -// Zero all statistics counters except those tracking cache size and number of -// files in the cache. -void zero_all_counters(const Config& config); - -// Collect cache statistics from all statistics counters. -std::pair collect_counters(const Config& config); - -// Format stats log in human-readable format. -std::string format_stats_log(const Config& config); - -// Format config header in human-readable format. -std::string format_config_header(const Config& config); - -// Format cache statistics in human-readable format. -std::string format_human_readable(const Counters& counters, - time_t last_updated, - bool from_log); - -// Format config footer in human-readable format. -std::string format_config_footer(const Config& config); - -// Format cache statistics in machine-readable format. -std::string format_machine_readable(const Counters& counters, - time_t last_updated); - -} // namespace Statistics diff --git a/src/ccache.cpp b/src/ccache.cpp index f13cfca7c..d586842a2 100644 --- a/src/ccache.cpp +++ b/src/ccache.cpp @@ -39,7 +39,6 @@ #include "ResultExtractor.hpp" #include "ResultRetriever.hpp" #include "SignalHandler.hpp" -#include "Statistics.hpp" #include "TemporaryFile.hpp" #include "UmaskScope.hpp" #include "Util.hpp" @@ -52,6 +51,8 @@ #include "language.hpp" #include +#include +#include #include #include #include @@ -1911,6 +1912,37 @@ configuration_printer(const std::string& key, static int cache_compilation(int argc, const char* const* argv); static Statistic do_cache_compilation(Context& ctx, const char* const* argv); +static void +log_result_to_debug_log(Context& ctx) +{ + if (ctx.config.log_file().empty() && !ctx.config.debug()) { + return; + } + + core::Statistics statistics(ctx.storage.primary().get_statistics_updates()); + const auto result_message = statistics.get_result_message(); + if (result_message) { + LOG("Result: {}", *result_message); + } +} + +static void +log_result_to_stats_log(Context& ctx) +{ + if (ctx.config.stats_log().empty()) { + return; + } + + core::Statistics statistics(ctx.storage.primary().get_statistics_updates()); + const auto result_id = statistics.get_result_id(); + if (!result_id) { + return; + } + + core::StatsLog(ctx.config.stats_log()) + .log_result(ctx.args_info.input_file, *result_id); +} + static void finalize_at_exit(Context& ctx) { @@ -1921,20 +1953,8 @@ finalize_at_exit(Context& ctx) return; } - if (!ctx.config.log_file().empty() || ctx.config.debug()) { - const auto result = ctx.storage.primary().get_result_message(); - if (result) { - LOG("Result: {}", *result); - } - } - - if (!ctx.config.stats_log().empty()) { - const auto result_id = ctx.storage.primary().get_result_id(); - if (result_id) { - Statistics::log_result( - ctx.config.stats_log(), ctx.args_info.input_file, *result_id); - } - } + log_result_to_debug_log(ctx); + log_result_to_stats_log(ctx); ctx.storage.finalize(); } catch (const core::ErrorBase& e) { @@ -2349,12 +2369,12 @@ handle_main_options(int argc, const char* const* argv) } case PRINT_STATS: { - Counters counters; + core::StatisticsCounters counters; time_t last_updated; std::tie(counters, last_updated) = - Statistics::collect_counters(ctx.config); - PRINT_RAW(stdout, - Statistics::format_machine_readable(counters, last_updated)); + ctx.storage.primary().get_all_statistics(); + core::Statistics statistics(counters); + PRINT_RAW(stdout, statistics.format_machine_readable(last_updated)); break; } @@ -2442,24 +2462,24 @@ handle_main_options(int argc, const char* const* argv) if (ctx.config.stats_log().empty()) { throw core::Fatal("No stats log has been configured"); } - PRINT_RAW(stdout, Statistics::format_stats_log(ctx.config)); - Counters counters = Statistics::read_log(ctx.config.stats_log()); - auto st = Stat::stat(ctx.config.stats_log(), Stat::OnError::log); - PRINT_RAW(stdout, - Statistics::format_human_readable(counters, st.mtime(), true)); + PRINT(stdout, "{:36}{}\n", "stats log", ctx.config.stats_log()); + core::Statistics statistics( + core::StatsLog(ctx.config.stats_log()).read()); + const auto timestamp = + Stat::stat(ctx.config.stats_log(), Stat::OnError::log).mtime(); + PRINT_RAW(stdout, statistics.format_human_readable(timestamp, true)); break; } case 's': { // --show-stats - PRINT_RAW(stdout, Statistics::format_config_header(ctx.config)); - Counters counters; + core::StatisticsCounters counters; time_t last_updated; std::tie(counters, last_updated) = - Statistics::collect_counters(ctx.config); - PRINT_RAW( - stdout, - Statistics::format_human_readable(counters, last_updated, false)); - PRINT_RAW(stdout, Statistics::format_config_footer(ctx.config)); + ctx.storage.primary().get_all_statistics(); + core::Statistics statistics(counters); + PRINT_RAW(stdout, statistics.format_config_header(ctx.config)); + PRINT_RAW(stdout, statistics.format_human_readable(last_updated, false)); + PRINT_RAW(stdout, statistics.format_config_footer(ctx.config)); break; } @@ -2492,7 +2512,7 @@ handle_main_options(int argc, const char* const* argv) } case 'z': // --zero-stats - Statistics::zero_all_counters(ctx.config); + ctx.storage.primary().zero_all_statistics(); PRINT_RAW(stdout, "Statistics zeroed\n"); break; diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 169216b4d..c282613fb 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -1,6 +1,8 @@ -# set( -# sources -# ${CMAKE_CURRENT_SOURCE_DIR}/file.cpp -# ) +set( + sources + ${CMAKE_CURRENT_SOURCE_DIR}/Statistics.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/StatisticsCounters.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/StatsLog.cpp +) -# target_sources(ccache_lib PRIVATE ${sources}) +target_sources(ccache_lib PRIVATE ${sources}) diff --git a/src/Statistics.cpp b/src/core/Statistics.cpp similarity index 57% rename from src/Statistics.cpp rename to src/core/Statistics.cpp index b9a1676e3..6c30fd3a2 100644 --- a/src/Statistics.cpp +++ b/src/core/Statistics.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020-2021 Joel Rosdahl and other contributors +// Copyright (C) 2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,113 +18,36 @@ #include "Statistics.hpp" -#include "AtomicFile.hpp" -#include "Config.hpp" -#include "File.hpp" -#include "Lockfile.hpp" -#include "Logging.hpp" -#include "Util.hpp" -#include "assertions.hpp" -#include "fmtmacros.hpp" +#include +#include +#include +#include -#include - -#include -#include - -const unsigned FLAG_NOZERO = 1; // don't zero with the -z option -const unsigned FLAG_ALWAYS = 2; // always show, even if zero -const unsigned FLAG_NEVER = 4; // never show -const unsigned FLAG_NOSTATSLOG = 8; // don't show for statslog +namespace core { using core::Statistic; -using nonstd::nullopt; -using nonstd::optional; // Returns a formatted version of a statistics value, or the empty string if the // statistics line shouldn't be printed. using FormatFunction = std::string (*)(uint64_t value); -static std::string -format_size(uint64_t size) -{ - return FMT("{:>11}", Util::format_human_readable_size(size)); -} - -static std::string -format_size_times_1024(uint64_t size) -{ - return format_size(size * 1024); -} - -static std::string -format_timestamp(uint64_t timestamp) -{ - if (timestamp > 0) { - const auto tm = Util::localtime(timestamp); - char buffer[100] = "?"; - if (tm) { - strftime(buffer, sizeof(buffer), "%c", &*tm); - } - return std::string(" ") + buffer; - } else { - return {}; - } -} - -static double -hit_rate(const Counters& counters) -{ - const uint64_t direct = counters.get(Statistic::direct_cache_hit); - const uint64_t preprocessed = counters.get(Statistic::preprocessed_cache_hit); - const uint64_t hit = direct + preprocessed; - const uint64_t miss = counters.get(Statistic::cache_miss); - const uint64_t total = hit + miss; - return total > 0 ? (100.0 * hit) / total : 0.0; -} - -static void -for_each_level_1_and_2_stats_file( - const std::string& cache_dir, - const std::function function) -{ - for (size_t level_1 = 0; level_1 <= 0xF; ++level_1) { - function(FMT("{}/{:x}/stats", cache_dir, level_1)); - for (size_t level_2 = 0; level_2 <= 0xF; ++level_2) { - function(FMT("{}/{:x}/{:x}/stats", cache_dir, level_1, level_2)); - } - } -} +static std::string format_size_times_1024(uint64_t value); +static std::string format_timestamp(uint64_t value); -std::pair -Statistics::collect_counters(const Config& config) -{ - Counters counters; - uint64_t zero_timestamp = 0; - time_t last_updated = 0; - - // Add up the stats in each directory. - for_each_level_1_and_2_stats_file(config.cache_dir(), [&](const auto& path) { - counters.set(Statistic::stats_zeroed_timestamp, 0); // Don't add - counters.increment(Statistics::read(path)); - zero_timestamp = - std::max(counters.get(Statistic::stats_zeroed_timestamp), zero_timestamp); - last_updated = std::max(last_updated, Stat::stat(path).mtime()); - }); - - counters.set(Statistic::stats_zeroed_timestamp, zero_timestamp); - return std::make_pair(counters, last_updated); -} +const unsigned FLAG_NOZERO = 1; // don't zero with the -z option +const unsigned FLAG_ALWAYS = 2; // always show, even if zero +const unsigned FLAG_NEVER = 4; // never show +const unsigned FLAG_NOSTATSLOG = 8; // don't show for statslog namespace { struct StatisticsField { - StatisticsField(Statistic statistic_, - const char* id_, - const char* message_, - unsigned flags_ = 0, - FormatFunction format_ = nullptr) + StatisticsField(const Statistic statistic_, + const char* const id_, + const char* const message_, + const unsigned flags_ = 0, + const FormatFunction format_ = nullptr) : statistic(statistic_), id(id_), message(message_), @@ -194,110 +117,51 @@ const StatisticsField k_statistics_fields[] = { STATISTICS_FIELD(none, nullptr), }; -namespace Statistics { - -Counters -read(const std::string& path) +static std::string +format_size(const uint64_t value) { - Counters counters; - - std::string data; - try { - data = Util::read_file(path); - } catch (const core::Error&) { - // Ignore. - return counters; - } - - size_t i = 0; - const char* str = data.c_str(); - while (true) { - char* end; - const uint64_t value = std::strtoull(str, &end, 10); - if (end == str) { - break; - } - counters.set_raw(i, value); - ++i; - str = end; - } - - return counters; + return FMT("{:>11}", Util::format_human_readable_size(value)); } -Counters -read_log(const std::string& path) +static std::string +format_size_times_1024(const uint64_t value) { - Counters counters; - - std::unordered_map m; - for (const auto& field : k_statistics_fields) { - m[field.id] = field.statistic; - } + return format_size(value * 1024); +} - std::ifstream in(path); - std::string line; - while (std::getline(in, line, '\n')) { - if (line[0] == '#') { - continue; - } - auto search = m.find(line); - if (search != m.end()) { - Statistic statistic = search->second; - counters.increment(statistic, 1); - } else { - LOG("Unknown statistic: {}", line); +static std::string +format_timestamp(const uint64_t value) +{ + if (value > 0) { + const auto tm = Util::localtime(value); + char buffer[100] = "?"; + if (tm) { + strftime(buffer, sizeof(buffer), "%c", &*tm); } + return std::string(" ") + buffer; + } else { + return {}; } - - return counters; } -optional -update(const std::string& path, - std::function function) +static double +hit_rate(const core::StatisticsCounters& counters) { - Lockfile lock(path); - if (!lock.acquired()) { - LOG("Failed to acquire lock for {}", path); - return nullopt; - } - - auto counters = Statistics::read(path); - function(counters); - - AtomicFile file(path, AtomicFile::Mode::text); - for (size_t i = 0; i < counters.size(); ++i) { - file.write(FMT("{}\n", counters.get_raw(i))); - } - try { - file.commit(); - } catch (const core::Error& e) { - // Make failure to write a stats file a soft error since it's not - // important enough to fail whole the process and also because it is - // called in the Context destructor. - LOG("Error: {}", e.what()); - } - - return counters; + const uint64_t direct = counters.get(Statistic::direct_cache_hit); + const uint64_t preprocessed = counters.get(Statistic::preprocessed_cache_hit); + const uint64_t hit = direct + preprocessed; + const uint64_t miss = counters.get(Statistic::cache_miss); + const uint64_t total = hit + miss; + return total > 0 ? (100.0 * hit) / total : 0.0; } -void -log_result(const std::string& path, - const std::string& input, - const std::string& result) +Statistics::Statistics(const StatisticsCounters& counters) + : m_counters(counters) { - File file(path, "ab"); - if (file) { - PRINT(*file, "# {}\n", input); - PRINT(*file, "{}\n", result); - } else { - LOG("Failed to open {}: {}", path, strerror(errno)); - } } static const StatisticsField* -get_result(const Counters& counters) +get_result(const core::StatisticsCounters& counters) { for (const auto& field : k_statistics_fields) { if (counters.get(field.statistic) != 0 && !(field.flags & FLAG_NOZERO)) { @@ -307,52 +171,28 @@ get_result(const Counters& counters) return nullptr; } -optional -get_result_id(const Counters& counters) +nonstd::optional +Statistics::get_result_id() const { - const auto result = get_result(counters); + const auto result = get_result(m_counters); if (result) { return result->id; } - return nullopt; + return nonstd::nullopt; } -optional -get_result_message(const Counters& counters) +nonstd::optional +Statistics::get_result_message() const { - const auto result = get_result(counters); + const auto result = get_result(m_counters); if (result) { return result->message; } - return nullopt; -} - -void -zero_all_counters(const Config& config) -{ - const time_t timestamp = time(nullptr); - - for_each_level_1_and_2_stats_file( - config.cache_dir(), [=](const std::string& path) { - Statistics::update(path, [=](Counters& cs) { - for (size_t i = 0; k_statistics_fields[i].message; ++i) { - if (!(k_statistics_fields[i].flags & FLAG_NOZERO)) { - cs.set(k_statistics_fields[i].statistic, 0); - } - } - cs.set(Statistic::stats_zeroed_timestamp, timestamp); - }); - }); -} - -std::string -format_stats_log(const Config& config) -{ - return FMT("{:36}{}\n", "stats log", config.stats_log()); + return nonstd::nullopt; } std::string -format_config_header(const Config& config) +Statistics::format_config_header(const Config& config) { std::string result; @@ -365,9 +205,8 @@ format_config_header(const Config& config) } std::string -format_human_readable(const Counters& counters, - time_t last_updated, - bool from_log) +Statistics::format_human_readable(const time_t last_updated, + const bool from_log) const { std::string result; @@ -387,7 +226,7 @@ format_human_readable(const Counters& counters, if (k_statistics_fields[i].flags & FLAG_NEVER) { continue; } - if (counters.get(statistic) == 0 + if (m_counters.get(statistic) == 0 && !(k_statistics_fields[i].flags & FLAG_ALWAYS)) { continue; } @@ -399,14 +238,14 @@ format_human_readable(const Counters& counters, const std::string value = k_statistics_fields[i].format - ? k_statistics_fields[i].format(counters.get(statistic)) - : FMT("{:8}", counters.get(statistic)); + ? k_statistics_fields[i].format(m_counters.get(statistic)) + : FMT("{:8}", m_counters.get(statistic)); if (!value.empty()) { result += FMT("{:32}{}\n", k_statistics_fields[i].message, value); } if (statistic == Statistic::cache_miss) { - double percent = hit_rate(counters); + double percent = hit_rate(m_counters); result += FMT("{:34}{:6.2f} %\n", "cache hit rate", percent); } } @@ -415,7 +254,7 @@ format_human_readable(const Counters& counters, } std::string -format_config_footer(const Config& config) +Statistics::format_config_footer(const Config& config) { std::string result; @@ -431,7 +270,7 @@ format_config_footer(const Config& config) } std::string -format_machine_readable(const Counters& counters, time_t last_updated) +Statistics::format_machine_readable(const time_t last_updated) const { std::string result; @@ -441,11 +280,33 @@ format_machine_readable(const Counters& counters, time_t last_updated) if (!(k_statistics_fields[i].flags & FLAG_NEVER)) { result += FMT("{}\t{}\n", k_statistics_fields[i].id, - counters.get(k_statistics_fields[i].statistic)); + m_counters.get(k_statistics_fields[i].statistic)); } } return result; } -} // namespace Statistics +std::unordered_map +Statistics::get_id_map() +{ + std::unordered_map result; + for (const auto& field : k_statistics_fields) { + result[field.id] = field.statistic; + } + return result; +} + +std::vector +Statistics::get_zeroable_fields() +{ + std::vector result; + for (const auto& field : k_statistics_fields) { + if (!(field.flags & FLAG_NOZERO)) { + result.push_back(field.statistic); + } + } + return result; +} + +} // namespace core diff --git a/src/core/Statistics.hpp b/src/core/Statistics.hpp new file mode 100644 index 000000000..46aa93559 --- /dev/null +++ b/src/core/Statistics.hpp @@ -0,0 +1,75 @@ +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include + +#include + +#include +#include + +class Config; + +namespace core { + +class Statistics +{ +public: + Statistics(const StatisticsCounters& counters); + + // Return a machine-readable string representing the final ccache result, or + // nullopt if there was no result. + nonstd::optional get_result_id() const; + + // Return a human-readable string representing the final ccache result, or + // nullopt if there was no result. + nonstd::optional get_result_message() const; + + // Format config header in human-readable format. + static std::string format_config_header(const Config& config); + + // Format cache statistics in human-readable format. + std::string format_human_readable(time_t last_updated, bool from_log) const; + + // Format config footer in human-readable format. + static std::string format_config_footer(const Config& config); + + // Format cache statistics in machine-readable format. + std::string format_machine_readable(time_t last_updated) const; + + const StatisticsCounters& counters() const; + + static std::unordered_map get_id_map(); + + static std::vector get_zeroable_fields(); + +private: + const StatisticsCounters m_counters; +}; + +// --- Inline implementations --- + +inline const StatisticsCounters& +Statistics::counters() const +{ + return m_counters; +} + +} // namespace core diff --git a/src/Counters.cpp b/src/core/StatisticsCounters.cpp similarity index 73% rename from src/Counters.cpp rename to src/core/StatisticsCounters.cpp index be80485af..2b57b34f6 100644 --- a/src/Counters.cpp +++ b/src/core/StatisticsCounters.cpp @@ -16,43 +16,44 @@ // this program; if not, write to the Free Software Foundation, Inc., 51 // Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -#include "Counters.hpp" +#include "StatisticsCounters.hpp" -#include "assertions.hpp" - -#include +#include #include -Counters::Counters() : m_counters(static_cast(core::Statistic::END)) +namespace core { + +StatisticsCounters::StatisticsCounters() + : m_counters(static_cast(Statistic::END)) { } uint64_t -Counters::get(core::Statistic statistic) const +StatisticsCounters::get(Statistic statistic) const { const auto index = static_cast(statistic); - ASSERT(index < static_cast(core::Statistic::END)); + ASSERT(index < static_cast(Statistic::END)); return index < m_counters.size() ? m_counters[index] : 0; } void -Counters::set(core::Statistic statistic, uint64_t value) +StatisticsCounters::set(Statistic statistic, uint64_t value) { const auto index = static_cast(statistic); - ASSERT(index < static_cast(core::Statistic::END)); + ASSERT(index < static_cast(Statistic::END)); m_counters[index] = value; } uint64_t -Counters::get_raw(size_t index) const +StatisticsCounters::get_raw(size_t index) const { ASSERT(index < size()); return m_counters[index]; } void -Counters::set_raw(size_t index, uint64_t value) +StatisticsCounters::set_raw(size_t index, uint64_t value) { if (index >= m_counters.size()) { m_counters.resize(index + 1); @@ -61,7 +62,7 @@ Counters::set_raw(size_t index, uint64_t value) } void -Counters::increment(core::Statistic statistic, int64_t value) +StatisticsCounters::increment(Statistic statistic, int64_t value) { const auto i = static_cast(statistic); if (i >= m_counters.size()) { @@ -73,7 +74,7 @@ Counters::increment(core::Statistic statistic, int64_t value) } void -Counters::increment(const Counters& other) +StatisticsCounters::increment(const StatisticsCounters& other) { m_counters.resize(std::max(size(), other.size())); for (size_t i = 0; i < other.size(); ++i) { @@ -84,14 +85,16 @@ Counters::increment(const Counters& other) } size_t -Counters::size() const +StatisticsCounters::size() const { return m_counters.size(); } bool -Counters::all_zero() const +StatisticsCounters::all_zero() const { return !std::any_of( m_counters.begin(), m_counters.end(), [](unsigned v) { return v != 0; }); } + +} // namespace core diff --git a/src/Counters.hpp b/src/core/StatisticsCounters.hpp similarity index 80% rename from src/Counters.hpp rename to src/core/StatisticsCounters.hpp index 5afabb564..f1f2c7243 100644 --- a/src/Counters.hpp +++ b/src/core/StatisticsCounters.hpp @@ -18,27 +18,29 @@ #pragma once -#include +#include "Statistic.hpp" #include #include #include +namespace core { + // A simple wrapper around a vector of integers used for the statistics // counters. -class Counters +class StatisticsCounters { public: - Counters(); + StatisticsCounters(); - uint64_t get(core::Statistic statistic) const; - void set(core::Statistic statistic, uint64_t value); + uint64_t get(Statistic statistic) const; + void set(Statistic statistic, uint64_t value); uint64_t get_raw(size_t index) const; void set_raw(size_t index, uint64_t value); - void increment(core::Statistic statistic, int64_t value = 1); - void increment(const Counters& other); + void increment(Statistic statistic, int64_t value = 1); + void increment(const StatisticsCounters& other); size_t size() const; @@ -48,3 +50,5 @@ public: private: std::vector m_counters; }; + +} // namespace core diff --git a/src/core/StatsLog.cpp b/src/core/StatsLog.cpp new file mode 100644 index 000000000..723189053 --- /dev/null +++ b/src/core/StatsLog.cpp @@ -0,0 +1,68 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "StatsLog.hpp" + +#include +#include +#include +#include + +#include + +namespace core { + +StatisticsCounters +StatsLog::read() const +{ + core::StatisticsCounters counters; + + const auto id_map = Statistics::get_id_map(); + + std::ifstream in(m_path); + std::string line; + while (std::getline(in, line, '\n')) { + if (line[0] == '#') { + continue; + } + const auto entry = id_map.find(line); + if (entry != id_map.end()) { + Statistic statistic = entry->second; + counters.increment(statistic, 1); + } else { + LOG("Unknown statistic: {}", line); + } + } + + return counters; +} + +void +StatsLog::log_result(const std::string& input_file, + const std::string& result_id) +{ + File file(m_path, "ab"); + if (file) { + PRINT(*file, "# {}\n", input_file); + PRINT(*file, "{}\n", result_id); + } else { + LOG("Failed to open {}: {}", m_path, strerror(errno)); + } +} + +} // namespace core diff --git a/src/core/StatsLog.hpp b/src/core/StatsLog.hpp new file mode 100644 index 000000000..4081b6733 --- /dev/null +++ b/src/core/StatsLog.hpp @@ -0,0 +1,45 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include "StatisticsCounters.hpp" + +#include + +namespace core { + +class StatsLog +{ +public: + StatsLog(const std::string& path); + + StatisticsCounters read() const; + void log_result(const std::string& input_file, const std::string& result_id); + +private: + const std::string m_path; +}; + +// --- Inline implementations --- + +inline StatsLog::StatsLog(const std::string& path) : m_path(path) +{ +} + +} // namespace core diff --git a/src/storage/primary/CMakeLists.txt b/src/storage/primary/CMakeLists.txt index 817da2c21..04a565e38 100644 --- a/src/storage/primary/CMakeLists.txt +++ b/src/storage/primary/CMakeLists.txt @@ -4,6 +4,8 @@ set( ${CMAKE_CURRENT_SOURCE_DIR}/PrimaryStorage.cpp ${CMAKE_CURRENT_SOURCE_DIR}/PrimaryStorage_cleanup.cpp ${CMAKE_CURRENT_SOURCE_DIR}/PrimaryStorage_compress.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/PrimaryStorage_statistics.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/StatsFile.cpp ${CMAKE_CURRENT_SOURCE_DIR}/util.cpp ) diff --git a/src/storage/primary/PrimaryStorage.cpp b/src/storage/primary/PrimaryStorage.cpp index b53ae0a7f..4aa9bb5fb 100644 --- a/src/storage/primary/PrimaryStorage.cpp +++ b/src/storage/primary/PrimaryStorage.cpp @@ -19,16 +19,14 @@ #include "PrimaryStorage.hpp" #include -#include #include #include -#include #include #include -#include #include #include #include +#include #include #ifdef HAVE_UNISTD_H @@ -132,8 +130,9 @@ PrimaryStorage::finalize() const auto bucket = getpid() % 256; const auto stats_file = FMT("{}/{:x}/{:x}/stats", m_config.cache_dir(), bucket / 16, bucket % 16); - Statistics::update( - stats_file, [&](auto& cs) { cs.increment(m_result_counter_updates); }); + StatsFile(stats_file).update([&](auto& cs) { + cs.increment(m_result_counter_updates); + }); return; } @@ -264,22 +263,6 @@ PrimaryStorage::increment_statistic(const Statistic statistic, m_result_counter_updates.increment(statistic, value); } -// Return a machine-readable string representing the final ccache result, or -// nullopt if there was no result. -nonstd::optional -PrimaryStorage::get_result_id() const -{ - return Statistics::get_result_id(m_result_counter_updates); -} - -// Return a human-readable string representing the final ccache result, or -// nullopt if there was no result. -nonstd::optional -PrimaryStorage::get_result_message() const -{ - return Statistics::get_result_message(m_result_counter_updates); -} - // Private methods PrimaryStorage::LookUpCacheFileResult @@ -330,11 +313,11 @@ PrimaryStorage::clean_internal_tempdir() }); } -nonstd::optional +nonstd::optional PrimaryStorage::update_stats_and_maybe_move_cache_file( const Digest& key, const std::string& current_path, - const Counters& counter_updates, + const core::StatisticsCounters& counter_updates, const core::CacheEntryType type) { if (counter_updates.all_zero()) { @@ -351,11 +334,11 @@ PrimaryStorage::update_stats_and_maybe_move_cache_file( if (!use_stats_on_level_1) { level_string += FMT("/{:x}", key.bytes()[0] & 0xF); } + const auto stats_file = FMT("{}/{}/stats", m_config.cache_dir(), level_string); - const auto counters = - Statistics::update(stats_file, [&counter_updates](auto& cs) { + StatsFile(stats_file).update([&counter_updates](auto& cs) { cs.increment(counter_updates); }); if (!counters) { diff --git a/src/storage/primary/PrimaryStorage.hpp b/src/storage/primary/PrimaryStorage.hpp index 675adf036..2361b1008 100644 --- a/src/storage/primary/PrimaryStorage.hpp +++ b/src/storage/primary/PrimaryStorage.hpp @@ -18,17 +18,15 @@ #pragma once -#include "util.hpp" - -#include #include +#include #include +#include #include #include class Config; -class Counters; namespace storage { namespace primary { @@ -57,24 +55,19 @@ public: void increment_statistic(core::Statistic statistic, int64_t value = 1); - // Return a machine-readable string representing the final ccache result, or - // nullopt if there was no result. - nonstd::optional get_result_id() const; + const core::StatisticsCounters& get_statistics_updates() const; + + // Zero all statistics counters except those tracking cache size and number of + // files in the cache. + void zero_all_statistics(); - // Return a human-readable string representing the final ccache result, or - // nullopt if there was no result. - nonstd::optional get_result_message() const; + // Get statistics and last time of update for the whole primary storage cache. + std::pair get_all_statistics() const; // --- Cleanup --- void clean_old(const ProgressReceiver& progress_receiver, uint64_t max_age); - void clean_dir(const std::string& subdir, - uint64_t max_size, - uint64_t max_files, - uint64_t max_age, - const ProgressReceiver& progress_receiver); - void clean_all(const ProgressReceiver& progress_receiver); void wipe_all(const ProgressReceiver& progress_receiver); @@ -92,11 +85,11 @@ private: // Main statistics updates (result statistics and size/count change for result // file) which get written into the statistics file belonging to the result // file. - Counters m_result_counter_updates; + core::StatisticsCounters m_result_counter_updates; // Statistics updates (only for manifest size/count change) which get written // into the statistics file belonging to the manifest. - Counters m_manifest_counter_updates; + core::StatisticsCounters m_manifest_counter_updates; // The manifest and result keys and paths are stored by put() so that // finalize() can use them to move the files in place. @@ -117,17 +110,32 @@ private: void clean_internal_tempdir(); - nonstd::optional - update_stats_and_maybe_move_cache_file(const Digest& key, - const std::string& current_path, - const Counters& counter_updates, - core::CacheEntryType type); + nonstd::optional + update_stats_and_maybe_move_cache_file( + const Digest& key, + const std::string& current_path, + const core::StatisticsCounters& counter_updates, + core::CacheEntryType type); // Join the cache directory, a '/' and `name` into a single path and return // it. Additionally, `level` single-character, '/'-separated subpaths are // split from the beginning of `name` before joining them all. std::string get_path_in_cache(uint8_t level, nonstd::string_view name) const; + + static void clean_dir(const std::string& subdir, + uint64_t max_size, + uint64_t max_files, + uint64_t max_age, + const ProgressReceiver& progress_receiver); }; +// --- Inline implementations --- + +inline const core::StatisticsCounters& +PrimaryStorage::get_statistics_updates() const +{ + return m_result_counter_updates; +} + } // namespace primary } // namespace storage diff --git a/src/storage/primary/PrimaryStorage_cleanup.cpp b/src/storage/primary/PrimaryStorage_cleanup.cpp index 51aeab16d..fd89cf035 100644 --- a/src/storage/primary/PrimaryStorage_cleanup.cpp +++ b/src/storage/primary/PrimaryStorage_cleanup.cpp @@ -22,9 +22,9 @@ #include #include #include -#include #include #include +#include #include #include @@ -65,7 +65,7 @@ update_counters(const std::string& dir, const bool cleanup_performed) { const std::string stats_file = dir + "/stats"; - Statistics::update(stats_file, [=](auto& cs) { + StatsFile(stats_file).update([=](auto& cs) { if (cleanup_performed) { cs.increment(Statistic::cleanups_performed); } diff --git a/src/storage/primary/PrimaryStorage_compress.cpp b/src/storage/primary/PrimaryStorage_compress.cpp index 400d1e5a9..5bd95be89 100644 --- a/src/storage/primary/PrimaryStorage_compress.cpp +++ b/src/storage/primary/PrimaryStorage_compress.cpp @@ -26,13 +26,13 @@ #include #include #include -#include #include #include #include #include #include #include +#include #include #include @@ -209,7 +209,7 @@ recompress_file(RecompressionStatistics& statistics, atomic_new_file.commit(); const auto new_stat = Stat::stat(cache_file.path(), Stat::OnError::log); - Statistics::update(stats_file, [=](auto& cs) { + StatsFile(stats_file).update([=](auto& cs) { cs.increment(core::Statistic::cache_size_kibibyte, Util::size_change_kibibyte(old_stat, new_stat)); }); diff --git a/src/storage/primary/PrimaryStorage_statistics.cpp b/src/storage/primary/PrimaryStorage_statistics.cpp new file mode 100644 index 000000000..3a2e015e5 --- /dev/null +++ b/src/storage/primary/PrimaryStorage_statistics.cpp @@ -0,0 +1,86 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "PrimaryStorage.hpp" + +#include +#include +#include +#include + +#include + +namespace storage { +namespace primary { + +static void +for_each_level_1_and_2_stats_file( + const std::string& cache_dir, + const std::function function) +{ + for (size_t level_1 = 0; level_1 <= 0xF; ++level_1) { + function(FMT("{}/{:x}/stats", cache_dir, level_1)); + for (size_t level_2 = 0; level_2 <= 0xF; ++level_2) { + function(FMT("{}/{:x}/{:x}/stats", cache_dir, level_1, level_2)); + } + } +} + +// Zero all statistics counters except those tracking cache size and number of +// files in the cache. +void +PrimaryStorage::zero_all_statistics() +{ + const time_t timestamp = time(nullptr); + const auto zeroable_fields = core::Statistics::get_zeroable_fields(); + + for_each_level_1_and_2_stats_file( + m_config.cache_dir(), [=](const std::string& path) { + StatsFile(path).update([=](core::StatisticsCounters& cs) { + for (const auto statistic : zeroable_fields) { + cs.set(statistic, 0); + } + cs.set(core::Statistic::stats_zeroed_timestamp, timestamp); + }); + }); +} + +// Get statistics and last time of update for the whole primary storage cache. +std::pair +PrimaryStorage::get_all_statistics() const +{ + core::StatisticsCounters counters; + uint64_t zero_timestamp = 0; + time_t last_updated = 0; + + // Add up the stats in each directory. + for_each_level_1_and_2_stats_file( + m_config.cache_dir(), [&](const auto& path) { + counters.set(core::Statistic::stats_zeroed_timestamp, 0); // Don't add + counters.increment(StatsFile(path).read()); + zero_timestamp = std::max( + counters.get(core::Statistic::stats_zeroed_timestamp), zero_timestamp); + last_updated = std::max(last_updated, Stat::stat(path).mtime()); + }); + + counters.set(core::Statistic::stats_zeroed_timestamp, zero_timestamp); + return std::make_pair(counters, last_updated); +} + +} // namespace primary +} // namespace storage diff --git a/src/storage/primary/StatsFile.cpp b/src/storage/primary/StatsFile.cpp new file mode 100644 index 000000000..87f0f881f --- /dev/null +++ b/src/storage/primary/StatsFile.cpp @@ -0,0 +1,94 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "StatsFile.hpp" + +#include +#include +#include +#include +#include +#include + +namespace storage { +namespace primary { + +StatsFile::StatsFile(const std::string& path) : m_path(path) +{ +} + +core::StatisticsCounters +StatsFile::read() const +{ + core::StatisticsCounters counters; + + std::string data; + try { + data = Util::read_file(m_path); + } catch (const core::Error&) { + // Ignore. + return counters; + } + + size_t i = 0; + const char* str = data.c_str(); + while (true) { + char* end; + const uint64_t value = std::strtoull(str, &end, 10); + if (end == str) { + break; + } + counters.set_raw(i, value); + ++i; + str = end; + } + + return counters; +} + +nonstd::optional +StatsFile::update( + std::function function) const +{ + Lockfile lock(m_path); + if (!lock.acquired()) { + LOG("Failed to acquire lock for {}", m_path); + return nonstd::nullopt; + } + + auto counters = read(); + function(counters); + + AtomicFile file(m_path, AtomicFile::Mode::text); + for (size_t i = 0; i < counters.size(); ++i) { + file.write(FMT("{}\n", counters.get_raw(i))); + } + try { + file.commit(); + } catch (const core::Error& e) { + // Make failure to write a stats file a soft error since it's not important + // enough to fail whole the process and also because it is called in the + // Context destructor. + LOG("Error: {}", e.what()); + } + + return counters; +} + +} // namespace primary +} // namespace storage diff --git a/src/storage/primary/StatsFile.hpp b/src/storage/primary/StatsFile.hpp new file mode 100644 index 000000000..0b8b96d4d --- /dev/null +++ b/src/storage/primary/StatsFile.hpp @@ -0,0 +1,51 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include + +#include + +#include +#include + +namespace storage { +namespace primary { + +class StatsFile +{ +public: + StatsFile(const std::string& path); + + // Read counters. No lock is acquired. If the file doesn't exist all returned + // counters will be zero. + core::StatisticsCounters read() const; + + // Acquire a lock, read counters, call `function` with the counters, write the + // counters and release the lock. Returns the resulting counters or nullopt on + // error (e.g. if the lock could not be acquired). + nonstd::optional + update(std::function) const; + +private: + const std::string m_path; +}; + +} // namespace primary +} // namespace storage diff --git a/unittest/CMakeLists.txt b/unittest/CMakeLists.txt index ada8b6887..6c2b3d2cc 100644 --- a/unittest/CMakeLists.txt +++ b/unittest/CMakeLists.txt @@ -6,21 +6,23 @@ set( test_AtomicFile.cpp test_Checksum.cpp test_Config.cpp - test_Counters.cpp test_Depfile.cpp test_FormatNonstdStringView.cpp test_Hash.cpp test_Lockfile.cpp test_NullCompression.cpp test_Stat.cpp - test_Statistics.cpp test_Util.cpp test_ZstdCompression.cpp test_argprocessing.cpp test_ccache.cpp test_compopt.cpp test_compression_types.cpp + test_core_Statistics.cpp + test_core_StatisticsCounters.cpp + test_core_StatsLog.cpp test_hashutil.cpp + test_storage_primary_StatsFile.cpp test_storage_primary_util.cpp test_util_Tokenizer.cpp test_util_expected.cpp diff --git a/unittest/test_core_Statistics.cpp b/unittest/test_core_Statistics.cpp new file mode 100644 index 000000000..2546c6791 --- /dev/null +++ b/unittest/test_core_Statistics.cpp @@ -0,0 +1,55 @@ +// Copyright (C) 2011-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "TestUtil.hpp" + +#include +#include + +#include + +#include // macOS bug: https://github.com/onqtam/doctest/issues/126 + +using core::Statistic; +using core::Statistics; +using core::StatisticsCounters; +using TestUtil::TestContext; + +TEST_SUITE_BEGIN("core::Statistics"); + +TEST_CASE("get_result_id") +{ + TestContext test_context; + + StatisticsCounters counters; + counters.increment(Statistic::cache_miss); + + CHECK(*Statistics(counters).get_result_id() == "cache_miss"); +} + +TEST_CASE("get_result_message") +{ + TestContext test_context; + + StatisticsCounters counters; + counters.increment(Statistic::cache_miss); + + CHECK(*Statistics(counters).get_result_message() == "cache miss"); +} + +TEST_SUITE_END(); diff --git a/unittest/test_Counters.cpp b/unittest/test_core_StatisticsCounters.cpp similarity index 92% rename from unittest/test_Counters.cpp rename to unittest/test_core_StatisticsCounters.cpp index e56710877..01db98972 100644 --- a/unittest/test_Counters.cpp +++ b/unittest/test_core_StatisticsCounters.cpp @@ -16,23 +16,24 @@ // this program; if not, write to the Free Software Foundation, Inc., 51 // Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -#include "../src/Counters.hpp" #include "TestUtil.hpp" #include +#include #include "third_party/doctest.h" using core::Statistic; +using core::StatisticsCounters; using TestUtil::TestContext; -TEST_SUITE_BEGIN("Counters"); +TEST_SUITE_BEGIN("core::StatisticsCounters"); -TEST_CASE("Counters") +TEST_CASE("StatisticsCounters") { TestContext test_context; - Counters counters; + StatisticsCounters counters; CHECK(counters.size() == static_cast(Statistic::END)); SUBCASE("Get and set statistic") @@ -77,7 +78,7 @@ TEST_CASE("Counters") counters.set(Statistic::files_in_cache, 10); counters.set(Statistic::cache_size_kibibyte, 1); - Counters updates; + StatisticsCounters updates; updates.set(Statistic::direct_cache_hit, 6); updates.set(Statistic::cache_miss, 5); updates.set(Statistic::files_in_cache, -1); diff --git a/unittest/test_core_StatsLog.cpp b/unittest/test_core_StatsLog.cpp new file mode 100644 index 000000000..175d9416f --- /dev/null +++ b/unittest/test_core_StatsLog.cpp @@ -0,0 +1,55 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "TestUtil.hpp" + +#include +#include + +#include + +using core::Statistic; +using core::StatsLog; +using TestUtil::TestContext; + +TEST_SUITE_BEGIN("core::StatsFile"); + +TEST_CASE("read") +{ + TestContext test_context; + + Util::write_file("stats.log", "# comment\ndirect_cache_hit\n"); + const auto counters = StatsLog("stats.log").read(); + + CHECK(counters.get(Statistic::direct_cache_hit) == 1); + CHECK(counters.get(Statistic::cache_miss) == 0); +} + +TEST_CASE("log_result") +{ + TestContext test_context; + + StatsLog stats_log("stats.log"); + stats_log.log_result("foo.c", "cache_miss"); + stats_log.log_result("bar.c", "preprocessed_cache_hit"); + + CHECK(Util::read_file("stats.log") + == "# foo.c\ncache_miss\n# bar.c\npreprocessed_cache_hit\n"); +} + +TEST_SUITE_END(); diff --git a/unittest/test_Statistics.cpp b/unittest/test_storage_primary_StatsFile.cpp similarity index 63% rename from unittest/test_Statistics.cpp rename to unittest/test_storage_primary_StatsFile.cpp index 21af4b4a4..2a7cdadb9 100644 --- a/unittest/test_Statistics.cpp +++ b/unittest/test_storage_primary_StatsFile.cpp @@ -16,25 +16,26 @@ // this program; if not, write to the Free Software Foundation, Inc., 51 // Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -#include "../src/Statistics.hpp" -#include "../src/Util.hpp" -#include "../src/fmtmacros.hpp" #include "TestUtil.hpp" +#include #include +#include +#include -#include "third_party/doctest.h" +#include using core::Statistic; +using storage::primary::StatsFile; using TestUtil::TestContext; -TEST_SUITE_BEGIN("Statistics"); +TEST_SUITE_BEGIN("storage::primary::StatsFile"); TEST_CASE("Read nonexistent") { TestContext test_context; - Counters counters = Statistics::read("test"); + const auto counters = StatsFile("test").read(); REQUIRE(counters.size() == static_cast(Statistic::END)); CHECK(counters.get(Statistic::cache_miss) == 0); @@ -45,7 +46,7 @@ TEST_CASE("Read bad") TestContext test_context; Util::write_file("test", "bad 1 2 3 4 5\n"); - Counters counters = Statistics::read("test"); + const auto counters = StatsFile("test").read(); REQUIRE(counters.size() == static_cast(Statistic::END)); CHECK(counters.get(Statistic::cache_miss) == 0); @@ -56,7 +57,7 @@ TEST_CASE("Read existing") TestContext test_context; Util::write_file("test", "0 1 2 3 27 5\n"); - Counters counters = Statistics::read("test"); + const auto counters = StatsFile("test").read(); REQUIRE(counters.size() == static_cast(Statistic::END)); CHECK(counters.get(Statistic::cache_miss) == 27); @@ -74,7 +75,7 @@ TEST_CASE("Read future counters") } Util::write_file("test", content); - Counters counters = Statistics::read("test"); + const auto counters = StatsFile("test").read(); REQUIRE(counters.size() == count); for (size_t i = 0; i < count; ++i) { @@ -82,24 +83,13 @@ TEST_CASE("Read future counters") } } -TEST_CASE("Read log") -{ - TestContext test_context; - - Util::write_file("stats.log", "# comment\ndirect_cache_hit\n"); - Counters counters = Statistics::read_log("stats.log"); - - CHECK(counters.get(Statistic::direct_cache_hit) == 1); - CHECK(counters.get(Statistic::cache_miss) == 0); -} - TEST_CASE("Update") { TestContext test_context; Util::write_file("test", "0 1 2 3 27 5\n"); - auto counters = Statistics::update("test", [](auto& cs) { + auto counters = StatsFile("test").update([](auto& cs) { cs.increment(Statistic::internal_error, 1); cs.increment(Statistic::cache_miss, 6); }); @@ -108,37 +98,9 @@ TEST_CASE("Update") CHECK(counters->get(Statistic::internal_error) == 4); CHECK(counters->get(Statistic::cache_miss) == 33); - counters = Statistics::read("test"); + counters = StatsFile("test").read(); CHECK(counters->get(Statistic::internal_error) == 4); CHECK(counters->get(Statistic::cache_miss) == 33); } -TEST_CASE("Get result") -{ - TestContext test_context; - - auto counters = Statistics::update( - "test", [](auto& cs) { cs.increment(Statistic::cache_miss, 1); }); - REQUIRE(counters); - - auto result = Statistics::get_result_message(*counters); - REQUIRE(result); -} - -TEST_CASE("Log result") -{ - TestContext test_context; - - auto counters = Statistics::update( - "test", [](auto& cs) { cs.increment(Statistic::cache_miss, 1); }); - REQUIRE(counters); - - auto result_id = Statistics::get_result_id(*counters); - REQUIRE(result_id); - Statistics::log_result("stats.log", "test.c", *result_id); - - auto statslog = Util::read_file("stats.log"); - REQUIRE(statslog.find(*result_id + "\n") != std::string::npos); -} - TEST_SUITE_END();