From: Joel Rosdahl Date: Fri, 11 Sep 2020 14:55:25 +0000 (+0200) Subject: Refactor flushing of stats and logs at ccache exit X-Git-Tag: v4.0~102 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=dd8f65aa5589709ca55bbb782050424a0010e8b8;p=thirdparty%2Fccache.git Refactor flushing of stats and logs at ccache exit Perform flushing of statistics counters (potentially triggering cache cleanup) and log files in a finalizer inside cache_compilation instead of in Conxtext’s destructor. This moves somewhat complex logic from Context into the main ccache code. As a side effect of this, stats_flush and stats_flush_to_file are superfluous. --- diff --git a/src/Context.cpp b/src/Context.cpp index c78e1254b..632750ba2 100644 --- a/src/Context.cpp +++ b/src/Context.cpp @@ -44,14 +44,7 @@ Context::Context() Context::~Context() { - stats_flush(*this); unlink_pending_tmp_files(); - - // Dump log buffer last to not lose any logs. - if (config.debug() && !args_info.output_obj.empty()) { - std::string path = fmt::format("{}.ccache-log", args_info.output_obj); - Logging::dump_log(path); - } } void diff --git a/src/ccache.cpp b/src/ccache.cpp index 7d3b39996..7917776c4 100644 --- a/src/ccache.cpp +++ b/src/ccache.cpp @@ -26,8 +26,10 @@ #include "Context.hpp" #include "Fd.hpp" #include "File.hpp" +#include "Finalizer.hpp" #include "FormatNonstdStringView.hpp" #include "Hash.hpp" +#include "Lockfile.hpp" #include "Logging.hpp" #include "Manifest.hpp" #include "MiniTrace.hpp" @@ -68,6 +70,7 @@ #endif #include +#include #include #ifndef MYNAME @@ -1937,54 +1940,136 @@ configuration_printer(const std::string& key, static int cache_compilation(int argc, const char* const* argv); static Statistic do_cache_compilation(Context& ctx, const char* const* argv); +static void +finalize_stats(const Context& ctx) +{ + const auto& config = ctx.config; + + if (config.disable()) { + // Just log result, don't update statistics. + log("Result: disabled"); + return; + } + + if (!config.log_file().empty() || config.debug()) { + const auto result = stats_get_result(ctx.counter_updates); + if (result) { + log("Result: {}", *result); + } + } + + if (!config.stats()) { + return; + } + + const auto counters = + Statistics::increment(ctx.stats_file(), ctx.counter_updates); + if (!counters) { + return; + } + + const std::string subdir(Util::dir_name(ctx.stats_file())); + bool need_cleanup = false; + + if (config.max_files() != 0 + && counters->get(Statistic::files_in_cache) > config.max_files() / 16) { + log("Need to clean up {} since it holds {} files (limit: {} files)", + subdir, + counters->get(Statistic::files_in_cache), + config.max_files() / 16); + need_cleanup = true; + } + if (config.max_size() != 0 + && counters->get(Statistic::cache_size_kibibyte) + > config.max_size() / 1024 / 16) { + log("Need to clean up {} since it holds {} KiB (limit: {} KiB)", + subdir, + counters->get(Statistic::cache_size_kibibyte), + config.max_size() / 1024 / 16); + need_cleanup = true; + } + + if (need_cleanup) { + const double factor = config.limit_multiple() / 16; + const uint64_t max_size = round(config.max_size() * factor); + const uint32_t max_files = round(config.max_files() * factor); + const time_t max_age = 0; + clean_up_dir( + subdir, max_size, max_files, max_age, [](double /*progress*/) {}); + } +} + +static void +finalize_at_exit(const Context& ctx) +{ + finalize_stats(ctx); + + // Dump log buffer last to not lose any logs. + if (ctx.config.debug() && !ctx.args_info.output_obj.empty()) { + const auto path = fmt::format("{}.ccache-log", ctx.args_info.output_obj); + Logging::dump_log(path); + } +} + // The entry point when invoked to cache a compilation. static int cache_compilation(int argc, const char* const* argv) { tzset(); // Needed for localtime_r. - auto ctx = std::make_unique(); - SignalHandler signal_handler(*ctx); - - initialize(*ctx, argc, argv); + bool fall_back_to_original_compiler = false; + Args saved_orig_args; - MTR_BEGIN("main", "find_compiler"); - find_compiler(*ctx, argv); - MTR_END("main", "find_compiler"); + { + Context ctx; + SignalHandler signal_handler(ctx); + Finalizer finalizer([&ctx] { finalize_at_exit(ctx); }); + + initialize(ctx, argc, argv); + + MTR_BEGIN("main", "find_compiler"); + find_compiler(ctx, argv); + MTR_END("main", "find_compiler"); + + try { + Statistic statistic = do_cache_compilation(ctx, argv); + ctx.counter_updates.increment(statistic); + } catch (const Failure& e) { + if (e.statistic() != Statistic::none) { + ctx.counter_updates.increment(e.statistic()); + } - try { - Statistic statistic = do_cache_compilation(*ctx, argv); - ctx->counter_updates.increment(statistic); - return EXIT_SUCCESS; - } catch (const Failure& e) { - if (e.statistic() != Statistic::none) { - ctx->counter_updates.increment(e.statistic()); - } + if (e.exit_code()) { + return *e.exit_code(); + } + // Else: Fall back to running the real compiler. + fall_back_to_original_compiler = true; - if (e.exit_code()) { - return *e.exit_code(); - } - // Else: Fall back to running the real compiler. + if (ctx.original_umask) { + umask(*ctx.original_umask); + } - if (ctx->original_umask) { - umask(*ctx->original_umask); - } + assert(!ctx.orig_args.empty()); - assert(!ctx->orig_args.empty()); + ctx.orig_args.erase_with_prefix("--ccache-"); + add_prefix(ctx, ctx.orig_args, ctx.config.prefix_command()); - ctx->orig_args.erase_with_prefix("--ccache-"); - add_prefix(*ctx, ctx->orig_args, ctx->config.prefix_command()); + log("Failed; falling back to running the real compiler"); - log("Failed; falling back to running the real compiler"); + saved_orig_args = std::move(ctx.orig_args); + auto execv_argv = saved_orig_args.to_argv(); + log("Executing {}", Util::format_argv_for_logging(execv_argv.data())); + // Run execv below after ctx and finalizer have been destructed. + } + } - Args saved_orig_args(std::move(ctx->orig_args)); + if (fall_back_to_original_compiler) { auto execv_argv = saved_orig_args.to_argv(); - - log("Executing {}", Util::format_argv_for_logging(execv_argv.data())); - ctx.reset(); // Dump debug logs last thing before executing. execv(execv_argv[0], const_cast(execv_argv.data())); throw Fatal("execv of {} failed: {}", execv_argv[0], strerror(errno)); } + + return EXIT_SUCCESS; } static Statistic diff --git a/src/cleanup.cpp b/src/cleanup.cpp index 9301108d0..5849fe6b1 100644 --- a/src/cleanup.cpp +++ b/src/cleanup.cpp @@ -31,7 +31,6 @@ #endif #include -#include using Logging::log; diff --git a/src/stats.cpp b/src/stats.cpp index 392653523..85684b4e2 100644 --- a/src/stats.cpp +++ b/src/stats.cpp @@ -34,13 +34,13 @@ #include "third_party/fmt/core.h" #include "third_party/nonstd/optional.hpp" -#include - const unsigned FLAG_NOZERO = 1; // don't zero with the -z option const unsigned FLAG_ALWAYS = 2; // always show, even if zero const unsigned FLAG_NEVER = 4; // never show using Logging::log; +using nonstd::nullopt; +using nonstd::optional; // Returns a formatted version of a statistics value, or the empty string if the // statistics line shouldn't be printed. @@ -189,85 +189,6 @@ stats_collect(const Config& config, Counters& counters, time_t* last_updated) counters.set(Statistic::stats_zeroed_timestamp, zero_timestamp); } -// Write counter updates in updates to sfile. -void -stats_flush_to_file(const Config& config, - const std::string& sfile, - const Counters& updates) -{ - if (updates.all_zero()) { - return; - } - - if (config.disable()) { - // Just log result, don't update statistics. - log("Result: disabled"); - return; - } - - if (!config.log_file().empty() || config.debug()) { - for (auto& field : k_statistics_fields) { - if (updates.get(field.statistic) != 0 && !(field.flags & FLAG_NOZERO)) { - log("Result: {}", field.message); - } - } - } - - if (!config.stats()) { - return; - } - - Counters counters; - - { - Lockfile lock(sfile); - if (!lock.acquired()) { - return; - } - - counters = Statistics::read(sfile); - counters.increment(updates); - Statistics::write(sfile, counters); - } - - std::string subdir(Util::dir_name(sfile)); - bool need_cleanup = false; - - if (config.max_files() != 0 - && counters.get(Statistic::files_in_cache) > config.max_files() / 16) { - log("Need to clean up {} since it holds {} files (limit: {} files)", - subdir, - counters.get(Statistic::files_in_cache), - config.max_files() / 16); - need_cleanup = true; - } - if (config.max_size() != 0 - && counters.get(Statistic::cache_size_kibibyte) - > config.max_size() / 1024 / 16) { - log("Need to clean up {} since it holds {} KiB (limit: {} KiB)", - subdir, - counters.get(Statistic::cache_size_kibibyte), - config.max_size() / 1024 / 16); - need_cleanup = true; - } - - if (need_cleanup) { - double factor = config.limit_multiple() / 16; - uint64_t max_size = round(config.max_size() * factor); - uint32_t max_files = round(config.max_files() * factor); - time_t max_age = 0; - clean_up_dir( - subdir, max_size, max_files, max_age, [](double /*progress*/) {}); - } -} - -// Write counter updates in counter_updates to disk. -void -stats_flush(Context& ctx) -{ - stats_flush_to_file(ctx.config, ctx.stats_file(), ctx.counter_updates); -} - // Sum and display the total stats for all cache dirs. void stats_summary(const Context& ctx) @@ -415,3 +336,14 @@ stats_add_cleanup(const std::string& dir, uint64_t count) updates.increment(Statistic::cleanups_performed, count); Statistics::increment(statsfile, updates); } + +optional +stats_get_result(const Counters& counters) +{ + for (const auto& field : k_statistics_fields) { + if (counters.get(field.statistic) != 0 && !(field.flags & FLAG_NOZERO)) { + return field.message; + } + } + return nullopt; +} diff --git a/src/stats.hpp b/src/stats.hpp index d8c1b813d..2f97add69 100644 --- a/src/stats.hpp +++ b/src/stats.hpp @@ -22,15 +22,13 @@ #include "Counters.hpp" +#include "third_party/nonstd/optional.hpp" + #include class Config; class Context; -void stats_flush(Context& ctx); -void stats_flush_to_file(const Config& config, - const std::string& sfile, - const Counters& updates); void stats_zero(const Context& ctx); void stats_summary(const Context& ctx); void stats_print(const Config& config); @@ -42,3 +40,5 @@ void stats_set_sizes(const std::string& dir, uint64_t num_files, uint64_t total_size); void stats_add_cleanup(const std::string& dir, uint64_t count); + +nonstd::optional stats_get_result(const Counters& counters);