From: Joel Rosdahl Date: Tue, 20 Sep 2022 17:01:38 +0000 (+0200) Subject: refactor: Use util::TimePoint for timestamps X-Git-Tag: v4.7~48 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=653dde812ed117f3ff03bf05d97d086482aac0c9;p=thirdparty%2Fccache.git refactor: Use util::TimePoint for timestamps --- diff --git a/cmake/GenerateConfigurationFile.cmake b/cmake/GenerateConfigurationFile.cmake index 7effeb86e..20be4eac1 100644 --- a/cmake/GenerateConfigurationFile.cmake +++ b/cmake/GenerateConfigurationFile.cmake @@ -29,7 +29,6 @@ set(functions geteuid getopt_long getpwuid - gettimeofday posix_fallocate realpath setenv diff --git a/cmake/config.h.in b/cmake/config.h.in index 9210147ed..28ea3c2a9 100644 --- a/cmake/config.h.in +++ b/cmake/config.h.in @@ -85,9 +85,6 @@ // Define if you have the "getpwuid" function. #cmakedefine HAVE_GETPWUID -// Define if you have the "gettimeofday" function. -#cmakedefine HAVE_GETTIMEOFDAY - // Define if you have the header file. #cmakedefine HAVE_LINUX_FS_H diff --git a/src/Context.cpp b/src/Context.cpp index 27a2a2c98..c46d36847 100644 --- a/src/Context.cpp +++ b/src/Context.cpp @@ -25,11 +25,9 @@ #include #include +#include #include -#ifdef HAVE_SYS_TIME_H -# include -#endif #ifdef HAVE_UNISTD_H # include #endif @@ -47,7 +45,7 @@ Context::Context() inode_cache(config) #endif { - gettimeofday(&time_of_invocation, nullptr); + time_of_invocation = util::TimePoint::now(); } void diff --git a/src/Context.hpp b/src/Context.hpp index a2e1ec0a7..63844489a 100644 --- a/src/Context.hpp +++ b/src/Context.hpp @@ -32,6 +32,7 @@ #include #include +#include #include #include @@ -40,10 +41,6 @@ #include #include -#ifdef HAVE_SYS_TIME_H -# include -#endif - class SignalHandler; class Context : NonCopyable @@ -69,11 +66,11 @@ public: Args orig_args; // Time of ccache invocation. - timeval time_of_invocation; + util::TimePoint time_of_invocation; // Time of compilation. Used to see if include files have changed after // compilation. - time_t time_of_compilation = 0; + util::TimePoint time_of_compilation; // Files included by the preprocessor and their hashes. std::unordered_map included_files; diff --git a/src/InodeCache.cpp b/src/InodeCache.cpp index 006ffab8c..8f09b9815 100644 --- a/src/InodeCache.cpp +++ b/src/InodeCache.cpp @@ -231,8 +231,8 @@ InodeCache::hash_inode(const std::string& path, key.st_dev = stat.device(); key.st_ino = stat.inode(); key.st_mode = stat.mode(); - key.st_mtim = stat.mtim(); - key.st_ctim = stat.ctim(); + key.st_mtim = stat.mtime().to_timespec(); + key.st_ctim = stat.ctime().to_timespec(); key.st_size = stat.size(); Hash hash; diff --git a/src/Logging.cpp b/src/Logging.cpp index 82e4b1c23..beac47072 100644 --- a/src/Logging.cpp +++ b/src/Logging.cpp @@ -35,9 +35,6 @@ #ifdef HAVE_SYSLOG_H # include #endif -#ifdef HAVE_SYS_TIME_H -# include -#endif #ifdef __linux__ # ifdef HAVE_SYS_IOCTL_H @@ -84,22 +81,21 @@ do_log(std::string_view message, bool bulk) if (!bulk) { char timestamp[100]; - struct timeval tv; - gettimeofday(&tv, nullptr); - auto tm = Util::localtime(tv.tv_sec); + auto now = util::TimePoint::now(); + auto tm = Util::localtime(now); if (tm) { strftime(timestamp, sizeof(timestamp), "%Y-%m-%dT%H:%M:%S", &*tm); } else { snprintf(timestamp, sizeof(timestamp), "%llu", - static_cast(tv.tv_sec)); + static_cast(now.sec())); } snprintf(prefix, sizeof(prefix), "[%s.%06d %-5d] ", timestamp, - static_cast(tv.tv_usec), + static_cast(now.nsec_decimal_part() / 1000), static_cast(getpid())); } diff --git a/src/MiniTrace.cpp b/src/MiniTrace.cpp index 07f495ad7..90ffcc4bd 100644 --- a/src/MiniTrace.cpp +++ b/src/MiniTrace.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020-2021 Joel Rosdahl and other contributors +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -24,10 +24,7 @@ #include "fmtmacros.hpp" #include - -#ifdef HAVE_SYS_TIME_H -# include -#endif +#include #ifdef HAVE_UNISTD_H # include @@ -53,18 +50,6 @@ get_system_tmp_dir() return "/tmp"; } -double -time_seconds() -{ -#ifdef HAVE_GETTIMEOFDAY - struct timeval tv; - gettimeofday(&tv, nullptr); - return (double)tv.tv_sec + (double)tv.tv_usec / 1000000.0; -#else - return (double)time(nullptr); -#endif -} - } // namespace MiniTrace::MiniTrace(const ArgsInfo& args_info) @@ -75,7 +60,8 @@ MiniTrace::MiniTrace(const ArgsInfo& args_info) m_tmp_trace_file = tmp_file.path; mtr_init(m_tmp_trace_file.c_str()); - m_start_time = FMT("{:f}", time_seconds()); + auto now = util::TimePoint::now(); + m_start_time = FMT("{}.{:06}", now.sec(), now.nsec_decimal_part() / 1000); MTR_INSTANT_C("", "", "time", m_start_time.c_str()); MTR_META_PROCESS_NAME("ccache"); MTR_START("program", "ccache", m_trace_id); diff --git a/src/Stat.hpp b/src/Stat.hpp index dcc4b2feb..5c81a5bad 100644 --- a/src/Stat.hpp +++ b/src/Stat.hpp @@ -19,6 +19,7 @@ #pragma once #include +#include #include #include @@ -125,9 +126,9 @@ public: dev_t device() const; ino_t inode() const; mode_t mode() const; - time_t atime() const; - time_t ctime() const; - time_t mtime() const; + util::TimePoint atime() const; + util::TimePoint ctime() const; + util::TimePoint mtime() const; uint64_t size() const; uint64_t size_on_disk() const; @@ -141,10 +142,6 @@ public: uint32_t reparse_tag() const; #endif - timespec atim() const; - timespec ctim() const; - timespec mtim() const; - protected: using StatFunction = int (*)(const char*, stat_t*); @@ -197,22 +194,40 @@ Stat::mode() const return m_stat.st_mode; } -inline time_t +inline util::TimePoint Stat::atime() const { - return atim().tv_sec; +#if defined(_WIN32) || defined(HAVE_STRUCT_STAT_ST_ATIM) + return util::TimePoint(m_stat.st_atim); +#elif defined(HAVE_STRUCT_STAT_ST_ATIMESPEC) + return util::TimePoint(m_stat.st_atimespec); +#else + return util::TimePoint(m_stat.st_atime, 0); +#endif } -inline time_t +inline util::TimePoint Stat::ctime() const { - return ctim().tv_sec; +#if defined(_WIN32) || defined(HAVE_STRUCT_STAT_ST_CTIM) + return util::TimePoint(m_stat.st_ctim); +#elif defined(HAVE_STRUCT_STAT_ST_CTIMESPEC) + return util::TimePoint(m_stat.st_ctimespec); +#else + return util::TimePoint(m_stat.st_ctime, 0); +#endif } -inline time_t +inline util::TimePoint Stat::mtime() const { - return mtim().tv_sec; +#if defined(_WIN32) || defined(HAVE_STRUCT_STAT_ST_MTIM) + return util::TimePoint(m_stat.st_mtim); +#elif defined(HAVE_STRUCT_STAT_ST_CTIMESPEC) + return util::TimePoint(m_stat.st_mtimespec); +#else + return util::TimePoint(m_stat.st_mtime, 0); +#endif } inline uint64_t @@ -262,39 +277,3 @@ Stat::reparse_tag() const return m_stat.st_reparse_tag; } #endif - -inline timespec -Stat::atim() const -{ -#if defined(_WIN32) || defined(HAVE_STRUCT_STAT_ST_ATIM) - return m_stat.st_atim; -#elif defined(HAVE_STRUCT_STAT_ST_ATIMESPEC) - return m_stat.st_atimespec; -#else - return {m_stat.st_atime, 0}; -#endif -} - -inline timespec -Stat::ctim() const -{ -#if defined(_WIN32) || defined(HAVE_STRUCT_STAT_ST_CTIM) - return m_stat.st_ctim; -#elif defined(HAVE_STRUCT_STAT_ST_CTIMESPEC) - return m_stat.st_ctimespec; -#else - return {m_stat.st_ctime, 0}; -#endif -} - -inline timespec -Stat::mtim() const -{ -#if defined(_WIN32) || defined(HAVE_STRUCT_STAT_ST_MTIM) - return m_stat.st_mtim; -#elif defined(HAVE_STRUCT_STAT_ST_CTIMESPEC) - return m_stat.st_mtimespec; -#else - return {m_stat.st_mtime, 0}; -#endif -} diff --git a/src/Util.cpp b/src/Util.cpp index 8b0b6e0ec..2d9bb3b73 100644 --- a/src/Util.cpp +++ b/src/Util.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -52,10 +53,6 @@ extern "C" { # include #endif -#ifdef HAVE_SYS_TIME_H -# include -#endif - #ifdef __linux__ # ifdef HAVE_SYS_IOCTL_H # include @@ -795,9 +792,9 @@ is_precompiled_header(std::string_view path) } std::optional -localtime(std::optional time) +localtime(std::optional time) { - time_t timestamp = time ? *time : ::time(nullptr); + time_t timestamp = time ? time->sec() : util::TimePoint::now().sec(); tm result; if (localtime_r(×tamp, &result)) { return result; diff --git a/src/Util.hpp b/src/Util.hpp index 2b2f0c1ff..e29aff3d4 100644 --- a/src/Util.hpp +++ b/src/Util.hpp @@ -19,6 +19,7 @@ #pragma once #include +#include #include #include @@ -235,7 +236,7 @@ bool is_precompiled_header(std::string_view path); // Thread-safe version of `localtime(3)`. If `time` is not specified the current // time of day is used. -std::optional localtime(std::optional time = {}); +std::optional localtime(std::optional time = {}); // Construct a normalized native path. // diff --git a/src/Win32Util.cpp b/src/Win32Util.cpp index 92d1ffb58..2769f48a3 100644 --- a/src/Win32Util.cpp +++ b/src/Win32Util.cpp @@ -133,22 +133,6 @@ get_last_ntstatus() } // namespace Win32Util -// From: https://stackoverflow.com/a/58162122/262458 -#ifdef _MSC_VER -int -gettimeofday(struct timeval* tp, struct timezone* /*tzp*/) -{ - namespace sc = std::chrono; - sc::system_clock::duration d = sc::system_clock::now().time_since_epoch(); - sc::seconds s = sc::duration_cast(d); - tp->tv_sec = static_cast(s.count()); - tp->tv_usec = - static_cast(sc::duration_cast(d - s).count()); - - return 0; -} -#endif - struct tm* localtime_r(time_t* _clock, struct tm* _result) { diff --git a/src/Win32Util.hpp b/src/Win32Util.hpp index c805dc7d8..2611759f8 100644 --- a/src/Win32Util.hpp +++ b/src/Win32Util.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020-2021 Joel Rosdahl and other contributors +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -27,7 +27,6 @@ struct tm* localtime_r(time_t* _clock, struct tm* _result); # ifdef _MSC_VER -int gettimeofday(struct timeval* tp, struct timezone* tzp); int asprintf(char** strp, const char* fmt, ...); # endif diff --git a/src/ccache.cpp b/src/ccache.cpp index 7bd3e4f3d..94d4a561a 100644 --- a/src/ccache.cpp +++ b/src/ccache.cpp @@ -65,9 +65,6 @@ #include #include -#ifdef HAVE_SYS_TIME_H -# include -#endif #ifdef HAVE_UNISTD_H # include #endif @@ -161,7 +158,7 @@ add_prefix(const Context& ctx, Args& args, const std::string& prefix_command) static std::string prepare_debug_path(const std::string& debug_dir, - const timeval& time_of_invocation, + const util::TimePoint& time_of_invocation, const std::string& output_obj, std::string_view suffix) { @@ -174,19 +171,19 @@ prepare_debug_path(const std::string& debug_dir, } char timestamp[100]; - const auto tm = Util::localtime(time_of_invocation.tv_sec); + const auto tm = Util::localtime(time_of_invocation); if (tm) { strftime(timestamp, sizeof(timestamp), "%Y%m%d_%H%M%S", &*tm); } else { snprintf(timestamp, sizeof(timestamp), "%llu", - static_cast(time_of_invocation.tv_sec)); + static_cast(time_of_invocation.sec())); } return FMT("{}.{}_{:06}.ccache-{}", prefix, timestamp, - time_of_invocation.tv_usec, + time_of_invocation.nsec_decimal_part() / 1000, suffix); } @@ -265,14 +262,14 @@ include_file_too_new(const Context& ctx, // starting compilation and writing the include file. See also the notes under // "Performance" in doc/MANUAL.adoc. if (!(ctx.config.sloppiness().is_enabled(core::Sloppy::include_file_mtime)) - && path_stat.mtime() >= ctx.time_of_compilation) { + && path_stat.mtime().sec() >= ctx.time_of_compilation.sec()) { LOG("Include file {} too new", path); return true; } // The same >= logic as above applies to the change time of the file. if (!(ctx.config.sloppiness().is_enabled(core::Sloppy::include_file_ctime)) - && path_stat.ctime() >= ctx.time_of_compilation) { + && path_stat.ctime().sec() >= ctx.time_of_compilation.sec()) { LOG("Include file {} ctime too new", path); return true; } @@ -799,8 +796,8 @@ update_manifest(Context& ctx, && ctx.time_of_compilation > std::max(stat.mtime(), stat.ctime()); return core::Manifest::FileStats{ stat.size(), - stat && cache_time ? stat.mtime() : -1, - stat && cache_time ? stat.ctime() : -1, + stat && cache_time ? stat.mtime() : util::TimePoint(), + stat && cache_time ? stat.ctime() : util::TimePoint(), }; }); if (added) { @@ -1020,7 +1017,7 @@ to_cache(Context& ctx, depend_mode_args.push_back(depend_extra_args); add_prefix(ctx, depend_mode_args, ctx.config.prefix_command()); - ctx.time_of_compilation = time(nullptr); + ctx.time_of_compilation = util::TimePoint::now(); result = do_execute(ctx, depend_mode_args); } MTR_END("execute", "compiler"); @@ -1101,7 +1098,7 @@ to_cache(Context& ctx, static nonstd::expected get_result_key_from_cpp(Context& ctx, Args& args, Hash& hash) { - ctx.time_of_compilation = time(nullptr); + ctx.time_of_compilation = util::TimePoint::now(); std::string preprocessed_path; std::string cpp_stderr_data; @@ -1192,7 +1189,7 @@ hash_compiler(const Context& ctx, } else if (ctx.config.compiler_check() == "mtime") { hash.hash_delimiter("cc_mtime"); hash.hash(st.size()); - hash.hash(st.mtime()); + hash.hash(st.mtime().nsec()); } else if (util::starts_with(ctx.config.compiler_check(), "string:")) { hash.hash_delimiter("cc_hash"); hash.hash(&ctx.config.compiler_check()[7]); @@ -1851,7 +1848,7 @@ calculate_result_and_manifest_key(Context& ctx, bool found_ccbin = false; hash.hash_delimiter("cache entry version"); - hash.hash(core::k_cache_entry_format_version); + hash.hash(core::CacheEntry::k_format_version); hash.hash_delimiter("result version"); hash.hash(core::Result::k_format_version); diff --git a/src/core/CacheEntry.cpp b/src/core/CacheEntry.cpp index fda2e1111..17e3253ab 100644 --- a/src/core/CacheEntry.cpp +++ b/src/core/CacheEntry.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -69,16 +70,25 @@ cache_entry_type_from_int(const uint8_t entry_type) namespace core { +// Version 0: +// - First version. +// Version 1: +// - Added self_contained field. +// - The checksum is now for the (potentially) compressed payload instead of +// the uncompressed payload, and the checksum is now always stored +// uncompressed. +const uint8_t CacheEntry::k_format_version = 1; + CacheEntry::Header::Header(const Config& config, core::CacheEntryType entry_type) : magic(k_ccache_magic), - entry_format_version(k_cache_entry_format_version), + entry_format_version(k_format_version), entry_type(entry_type), compression_type(compression_type_from_config(config)), compression_level(compression_level_from_config(config)), self_contained(entry_type != CacheEntryType::result || !core::Result::Serializer::use_raw_files(config)), - creation_time(time(nullptr)), + creation_time(util::TimePoint::now().sec()), ccache_version(CCACHE_VERSION), namespace_(config.namespace_()), entry_size(0) @@ -132,7 +142,7 @@ CacheEntry::Header::parse(nonstd::span data) } reader.read_int(entry_format_version); - if (entry_format_version != core::k_cache_entry_format_version) { + if (entry_format_version != k_format_version) { throw core::Error( FMT("Unknown entry format version: {}", entry_format_version)); } diff --git a/src/core/CacheEntry.hpp b/src/core/CacheEntry.hpp index c4adecfc4..0402f3a9f 100644 --- a/src/core/CacheEntry.hpp +++ b/src/core/CacheEntry.hpp @@ -63,18 +63,10 @@ namespace core { const uint16_t k_ccache_magic = 0xccac; -// Version 0: -// - First version. -// Version 1: -// - Added self_contained field. -// - The checksum is now for the (potentially) compressed payload instead of -// the uncompressed payload, and the checksum is now always stored -// uncompressed. -const uint16_t k_cache_entry_format_version = 1; - class CacheEntry { public: + static const uint8_t k_format_version; constexpr static uint8_t default_compression_level = 1; class Header diff --git a/src/core/Manifest.cpp b/src/core/Manifest.cpp index 7922be7c9..cf4f4fccd 100644 --- a/src/core/Manifest.cpp +++ b/src/core/Manifest.cpp @@ -76,7 +76,7 @@ template<> struct hash namespace core { -const uint8_t Manifest::k_format_version = 0; +const uint8_t Manifest::k_format_version = 1; void Manifest::read(nonstd::span data) @@ -107,8 +107,8 @@ Manifest::read(nonstd::span data) reader.read_int(entry.index); reader.read_and_copy_bytes({entry.digest.bytes(), Digest::size()}); reader.read_int(entry.fsize); - reader.read_int(entry.mtime); - reader.read_int(entry.ctime); + entry.mtime.set_sec(reader.read_int()); + entry.ctime.set_sec(reader.read_int()); } const auto result_count = reader.read_int(); @@ -264,8 +264,8 @@ Manifest::serialize(util::Bytes& output) writer.write_int(file_info.index); writer.write_bytes({file_info.digest.bytes(), Digest::size()}); writer.write_int(file_info.fsize); - writer.write_int(file_info.mtime); - writer.write_int(file_info.ctime); + writer.write_int(file_info.mtime.sec()); + writer.write_int(file_info.ctime.sec()); } writer.write_int(m_results.size()); @@ -430,8 +430,16 @@ Manifest::inspect(FILE* const stream) const PRINT(stream, " Path index: {}\n", m_file_infos[i].index); PRINT(stream, " Hash: {}\n", m_file_infos[i].digest.to_string()); PRINT(stream, " File size: {}\n", m_file_infos[i].fsize); - PRINT(stream, " Mtime: {}\n", m_file_infos[i].mtime); - PRINT(stream, " Ctime: {}\n", m_file_infos[i].ctime); + if (m_file_infos[i].mtime == util::TimePoint(-1)) { + PRINT_RAW(stream, " Mtime: -\n"); + } else { + PRINT(stream, " Mtime: {}\n", m_file_infos[i].mtime.sec()); + } + if (m_file_infos[i].ctime == util::TimePoint(-1)) { + PRINT_RAW(stream, " Ctime: -\n"); + } else { + PRINT(stream, " Ctime: {}\n", m_file_infos[i].ctime.sec()); + } } PRINT(stream, "Results ({}):\n", m_results.size()); diff --git a/src/core/Manifest.hpp b/src/core/Manifest.hpp index 399194bc4..a8f3b67e7 100644 --- a/src/core/Manifest.hpp +++ b/src/core/Manifest.hpp @@ -20,6 +20,7 @@ #include #include +#include #include @@ -42,8 +43,8 @@ public: struct FileStats { uint64_t size; - int64_t mtime; - int64_t ctime; + util::TimePoint mtime; + util::TimePoint ctime; }; using FileStater = std::function; @@ -67,11 +68,11 @@ public: private: struct FileInfo { - uint32_t index; // Index to m_files. - Digest digest; // Digest of referenced file. - uint64_t fsize; // Size of referenced file. - int64_t mtime; // mtime of referenced file. - int64_t ctime; // ctime of referenced file. + uint32_t index; // Index to m_files. + Digest digest; // Digest of referenced file. + uint64_t fsize; // Size of referenced file. + util::TimePoint mtime; // mtime of referenced file. + util::TimePoint ctime; // ctime of referenced file. bool operator==(const FileInfo& other) const; }; diff --git a/src/core/Statistics.cpp b/src/core/Statistics.cpp index ab7d5e4bd..289b77222 100644 --- a/src/core/Statistics.cpp +++ b/src/core/Statistics.cpp @@ -127,9 +127,9 @@ static_assert(sizeof(k_statistics_fields) / sizeof(k_statistics_fields[0]) == static_cast(Statistic::END) - 1); static std::string -format_timestamp(const uint64_t value) +format_timestamp(const util::TimePoint& value) { - if (value == 0) { + if (value.sec() == 0) { return "never"; } else { const auto tm = Util::localtime(value); @@ -219,7 +219,7 @@ add_ratio_row(util::TextTable& table, std::string Statistics::format_human_readable(const Config& config, - const time_t last_updated, + const util::TimePoint& last_updated, const uint8_t verbosity, const bool from_log) const { @@ -251,7 +251,7 @@ Statistics::format_human_readable(const Config& config, table.add_row( {"Stats updated:", C(format_timestamp(last_updated)).colspan(4)}); if (verbosity > 1) { - const uint64_t last_zeroed = S(stats_zeroed_timestamp); + const util::TimePoint last_zeroed(S(stats_zeroed_timestamp)); table.add_row( {"Stats zeroed:", C(format_timestamp(last_zeroed)).colspan(4)}); } @@ -351,11 +351,11 @@ Statistics::format_human_readable(const Config& config, } std::string -Statistics::format_machine_readable(const time_t last_updated) const +Statistics::format_machine_readable(const util::TimePoint& last_updated) const { std::vector lines; - lines.push_back(FMT("stats_updated_timestamp\t{}\n", last_updated)); + lines.push_back(FMT("stats_updated_timestamp\t{}\n", last_updated.sec())); for (const auto& field : k_statistics_fields) { if (!(field.flags & FLAG_NEVER)) { diff --git a/src/core/Statistics.hpp b/src/core/Statistics.hpp index eb80e1cae..f74d81e52 100644 --- a/src/core/Statistics.hpp +++ b/src/core/Statistics.hpp @@ -19,6 +19,7 @@ #pragma once #include +#include #include #include @@ -40,12 +41,13 @@ public: // Format cache statistics in human-readable format. std::string format_human_readable(const Config& config, - time_t last_updated, + const util::TimePoint& last_updated, uint8_t verbosity, bool from_log) const; // Format cache statistics in machine-readable format. - std::string format_machine_readable(time_t last_updated) const; + std::string + format_machine_readable(const util::TimePoint& last_updated) const; const StatisticsCounters& counters() const; diff --git a/src/core/mainoptions.cpp b/src/core/mainoptions.cpp index 717ca8e5c..0fbe2e9a8 100644 --- a/src/core/mainoptions.cpp +++ b/src/core/mainoptions.cpp @@ -283,11 +283,8 @@ trim_dir(const std::string& dir, }); std::sort(files.begin(), files.end(), [&](const auto& f1, const auto& f2) { - const auto ts_1 = trim_lru_mtime ? f1.stat.mtim() : f1.stat.atim(); - const auto ts_2 = trim_lru_mtime ? f2.stat.mtim() : f2.stat.atim(); - const auto ns_1 = 1'000'000'000ULL * ts_1.tv_sec + ts_1.tv_nsec; - const auto ns_2 = 1'000'000'000ULL * ts_2.tv_sec + ts_2.tv_nsec; - return ns_1 < ns_2; + return trim_lru_mtime ? f1.stat.mtime() < f2.stat.mtime() + : f1.stat.atime() < f2.stat.atime(); }); uint64_t size_after = size_before; @@ -506,9 +503,7 @@ process_main_options(int argc, const char* const* argv) return inspect_path(arg); case PRINT_STATS: { - StatisticsCounters counters; - time_t last_updated; - std::tie(counters, last_updated) = + const auto [counters, last_updated] = storage::primary::PrimaryStorage(config).get_all_statistics(); Statistics statistics(counters); PRINT_RAW(stdout, statistics.format_machine_readable(last_updated)); @@ -600,9 +595,7 @@ process_main_options(int argc, const char* const* argv) } case 's': { // --show-stats - StatisticsCounters counters; - time_t last_updated; - std::tie(counters, last_updated) = + const auto [counters, last_updated] = storage::primary::PrimaryStorage(config).get_all_statistics(); Statistics statistics(counters); PRINT_RAW(stdout, diff --git a/src/storage/primary/PrimaryStorage.cpp b/src/storage/primary/PrimaryStorage.cpp index 32c7f581e..88154bfff 100644 --- a/src/storage/primary/PrimaryStorage.cpp +++ b/src/storage/primary/PrimaryStorage.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #ifdef HAVE_UNISTD_H @@ -40,7 +41,7 @@ namespace storage::primary { // How often (in seconds) to scan $CCACHE_DIR/tmp for left-over temporary // files. -const int k_tempdir_cleanup_interval = 2 * 24 * 60 * 60; // 2 days +const util::Duration k_tempdir_cleanup_interval(2 * 24 * 60 * 60); // 2 days // Maximum files per cache directory. This constant is somewhat arbitrarily // chosen to be large enough to avoid unnecessary cache levels but small enough @@ -371,7 +372,7 @@ PrimaryStorage::clean_internal_tempdir() { MTR_SCOPE("primary_storage", "clean_internal_tempdir"); - const time_t now = time(nullptr); + const auto now = util::TimePoint::now(); const auto cleaned_stamp = FMT("{}/.cleaned", m_config.temporary_dir()); const auto cleaned_stat = Stat::stat(cleaned_stamp); if (cleaned_stat diff --git a/src/storage/primary/PrimaryStorage.hpp b/src/storage/primary/PrimaryStorage.hpp index 2789a141d..966f04e1e 100644 --- a/src/storage/primary/PrimaryStorage.hpp +++ b/src/storage/primary/PrimaryStorage.hpp @@ -25,6 +25,7 @@ #include #include #include +#include #include @@ -85,7 +86,8 @@ public: void zero_all_statistics(); // Get statistics and last time of update for the whole primary storage cache. - std::pair get_all_statistics() const; + std::pair + get_all_statistics() const; // --- Cleanup --- diff --git a/src/storage/primary/PrimaryStorage_cleanup.cpp b/src/storage/primary/PrimaryStorage_cleanup.cpp index 6165c724d..060720c38 100644 --- a/src/storage/primary/PrimaryStorage_cleanup.cpp +++ b/src/storage/primary/PrimaryStorage_cleanup.cpp @@ -109,7 +109,7 @@ PrimaryStorage::clean_dir(const std::string& subdir, uint64_t cache_size = 0; uint64_t files_in_cache = 0; - time_t current_time = time(nullptr); + auto current_time = util::TimePoint::now(); std::unordered_map /*associated_raw_files*/> raw_files_map; @@ -124,7 +124,7 @@ PrimaryStorage::clean_dir(const std::string& subdir, } // Delete any tmp files older than 1 hour right away. - if (file.lstat().mtime() + 3600 < current_time + if (file.lstat().mtime() + util::Duration(3600) < current_time && TemporaryFile::is_tmp_file(file.path())) { Util::unlink_tmp(file.path()); continue; @@ -142,11 +142,7 @@ PrimaryStorage::clean_dir(const std::string& subdir, // Sort according to modification time, oldest first. std::sort(files.begin(), files.end(), [](const auto& f1, const auto& f2) { - const auto ts_1 = f1.lstat().mtim(); - const auto ts_2 = f2.lstat().mtim(); - const auto ns_1 = 1'000'000'000ULL * ts_1.tv_sec + ts_1.tv_nsec; - const auto ns_2 = 1'000'000'000ULL * ts_2.tv_sec + ts_2.tv_nsec; - return ns_1 < ns_2; + return f1.lstat().mtime() < f2.lstat().mtime(); }); LOG("Before cleanup: {:.0f} KiB, {:.0f} files", @@ -165,8 +161,7 @@ PrimaryStorage::clean_dir(const std::string& subdir, if ((max_size == 0 || cache_size <= max_size) && (max_files == 0 || files_in_cache <= max_files) && (!max_age - || file.lstat().mtime() - > (current_time - static_cast(*max_age))) + || file.lstat().mtime() > (current_time - util::Duration(*max_age))) && (!namespace_ || max_age)) { break; } diff --git a/src/storage/primary/PrimaryStorage_compress.cpp b/src/storage/primary/PrimaryStorage_compress.cpp index bceed4555..b9ee286f1 100644 --- a/src/storage/primary/PrimaryStorage_compress.cpp +++ b/src/storage/primary/PrimaryStorage_compress.cpp @@ -136,7 +136,7 @@ recompress_file(RecompressionStatistics& statistics, core::CacheEntry cache_entry(cache_file_data); cache_entry.verify_checksum(); - header.entry_format_version = core::k_cache_entry_format_version; + header.entry_format_version = core::CacheEntry::k_format_version; header.compression_type = level ? core::CompressionType::zstd : core::CompressionType::none; header.compression_level = wanted_level; @@ -147,7 +147,7 @@ recompress_file(RecompressionStatistics& statistics, new_cache_file.commit(); // Restore mtime/atime to keep cache LRU cleanup working as expected: - util::set_timestamps(cache_file.path(), old_stat.mtim(), old_stat.atim()); + util::set_timestamps(cache_file.path(), old_stat.mtime(), old_stat.atime()); const auto new_stat = Stat::stat(cache_file.path(), Stat::OnError::log); StatsFile(stats_file).update([=](auto& cs) { diff --git a/src/storage/primary/PrimaryStorage_statistics.cpp b/src/storage/primary/PrimaryStorage_statistics.cpp index dc9ff146c..833e13ddb 100644 --- a/src/storage/primary/PrimaryStorage_statistics.cpp +++ b/src/storage/primary/PrimaryStorage_statistics.cpp @@ -45,7 +45,7 @@ for_each_level_1_and_2_stats_file( void PrimaryStorage::zero_all_statistics() { - const time_t timestamp = time(nullptr); + const auto now = util::TimePoint::now(); const auto zeroable_fields = core::Statistics::get_zeroable_fields(); for_each_level_1_and_2_stats_file( @@ -54,18 +54,18 @@ PrimaryStorage::zero_all_statistics() for (const auto statistic : zeroable_fields) { cs.set(statistic, 0); } - cs.set(core::Statistic::stats_zeroed_timestamp, timestamp); + cs.set(core::Statistic::stats_zeroed_timestamp, now.sec()); }); }); } // Get statistics and last time of update for the whole primary storage cache. -std::pair +std::pair PrimaryStorage::get_all_statistics() const { core::StatisticsCounters counters; uint64_t zero_timestamp = 0; - time_t last_updated = 0; + util::TimePoint last_updated; // Add up the stats in each directory. for_each_level_1_and_2_stats_file( diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 59d72f6ac..f3133e715 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -3,6 +3,7 @@ set( Bytes.cpp LockFile.cpp TextTable.cpp + TimePoint.cpp Tokenizer.cpp file.cpp path.cpp diff --git a/src/util/LockFile.cpp b/src/util/LockFile.cpp index 3a0bd7978..0d7e96e8e 100644 --- a/src/util/LockFile.cpp +++ b/src/util/LockFile.cpp @@ -29,9 +29,6 @@ #include "third_party/fmt/core.h" -#ifdef HAVE_SYS_TIME_H -# include -#endif #ifdef HAVE_UNISTD_H # include #endif @@ -44,10 +41,8 @@ const double k_min_sleep_time = 0.010; const double k_max_sleep_time = 0.050; #ifndef _WIN32 -const double k_staleness_limit = 2; -const double k_keep_alive_interval = k_staleness_limit / 4; -const auto k_keep_alive_interval_ms = std::chrono::milliseconds{ - static_cast(k_keep_alive_interval * 1000)}; +const util::Duration k_staleness_limit(2); +const util::Duration k_keep_alive_interval(k_staleness_limit / 4); #endif namespace { @@ -155,21 +150,6 @@ LockFile::acquire(const bool blocking) #ifndef _WIN32 -static double -time_from_clock() -{ - timeval tv; - gettimeofday(&tv, nullptr); - return tv.tv_sec + static_cast(tv.tv_usec) / 1'000'000; -} - -static double -time_from_stat(const Stat& stat) -{ - const auto mtime = stat.mtim(); - return mtime.tv_sec + static_cast(mtime.tv_nsec) / 1'000'000'000; -} - bool LockFile::do_acquire(const bool blocking) { @@ -178,9 +158,9 @@ LockFile::do_acquire(const bool blocking) << std::this_thread::get_id(); const auto content_prefix = ss.str(); - double last_seen_activity = [this] { + util::TimePoint last_seen_activity = [this] { const auto last_lock_update = get_last_lock_update(); - return last_lock_update ? *last_lock_update : time_from_clock(); + return last_lock_update ? *last_lock_update : util::TimePoint::now(); }(); std::string initial_content; @@ -188,7 +168,9 @@ LockFile::do_acquire(const bool blocking) k_max_sleep_time * 1000); while (true) { - const auto my_content = FMT("{}-{}", content_prefix, time_from_clock()); + const auto now = util::TimePoint::now(); + const auto my_content = + FMT("{}-{}.{}", content_prefix, now.sec(), now.nsec()); if (symlink(my_content.c_str(), m_lock_file.c_str()) == 0) { // We got the lock. @@ -245,20 +227,23 @@ LockFile::do_acquire(const bool blocking) last_seen_activity = std::max(last_seen_activity, *last_lock_update); } - const double inactive_duration = time_from_clock() - last_seen_activity; + const util::Duration inactive_duration = + util::TimePoint::now() - last_seen_activity; if (inactive_duration < k_staleness_limit) { - LOG("Lock {} held by another process active {:.3f} seconds ago", + LOG("Lock {} held by another process active {}.{:03} seconds ago", m_lock_file, - inactive_duration); + inactive_duration.sec(), + inactive_duration.nsec() / 1'000'000); if (!blocking) { return false; } } else if (content == initial_content) { // The lock seems to be stale -- break it and try again. - LOG("Breaking {} since it has been inactive for {:.3f} seconds", + LOG("Breaking {} since it has been inactive for {}.{:03} seconds", m_lock_file, - inactive_duration); + inactive_duration.sec(), + inactive_duration.nsec() / 1'000'000); if (!on_before_break() || !Util::unlink_tmp(m_lock_file)) { return false; } @@ -376,7 +361,10 @@ LongLivedLockFile::on_after_acquire() while (true) { std::unique_lock lock(m_stop_keep_alive_mutex); m_stop_keep_alive_condition.wait_for( - lock, k_keep_alive_interval_ms, [this] { return m_stop_keep_alive; }); + lock, + std::chrono::seconds(k_keep_alive_interval.sec()) + + std::chrono::nanoseconds(k_keep_alive_interval.nsec()), + [this] { return m_stop_keep_alive; }); if (m_stop_keep_alive) { return; } @@ -407,11 +395,11 @@ LongLivedLockFile::on_before_break() return Util::unlink_tmp(m_alive_file); } -std::optional +std::optional LongLivedLockFile::get_last_lock_update() { if (const auto stat = Stat::stat(m_alive_file); stat) { - return time_from_stat(stat); + return stat.mtime(); } else { return std::nullopt; } diff --git a/src/util/LockFile.hpp b/src/util/LockFile.hpp index 1bf6808cb..705dd7b87 100644 --- a/src/util/LockFile.hpp +++ b/src/util/LockFile.hpp @@ -19,6 +19,7 @@ #pragma once #include +#include #include #include @@ -62,7 +63,7 @@ private: #ifndef _WIN32 bool do_acquire(bool blocking); virtual bool on_before_break(); - virtual std::optional get_last_lock_update(); + virtual std::optional get_last_lock_update(); #else void* do_acquire(bool blocking); #endif @@ -98,7 +99,7 @@ private: void on_after_acquire() override; void on_before_release() override; bool on_before_break() override; - std::optional get_last_lock_update() override; + std::optional get_last_lock_update() override; #endif }; @@ -134,7 +135,7 @@ LockFile::on_before_break() return true; } -inline std::optional +inline std::optional LockFile::get_last_lock_update() { return std::nullopt; diff --git a/src/util/file.cpp b/src/util/file.cpp index 728f521e9..b3d3771fc 100644 --- a/src/util/file.cpp +++ b/src/util/file.cpp @@ -42,6 +42,10 @@ # endif #endif +#include +#include +#include + #include #include #include @@ -238,30 +242,31 @@ read_file_part(const std::string& path, size_t pos, size_t count); void set_timestamps(const std::string& path, - std::optional mtime, - std::optional atime) + std::optional mtime, + std::optional atime) { #ifdef HAVE_UTIMENSAT timespec atime_mtime[2]; if (mtime) { - atime_mtime[0] = atime ? *atime : *mtime; - atime_mtime[1] = *mtime; + atime_mtime[0] = (atime ? *atime : *mtime).to_timespec(); + atime_mtime[1] = mtime->to_timespec(); } utimensat(AT_FDCWD, path.c_str(), mtime ? atime_mtime : nullptr, 0); #elif defined(HAVE_UTIMES) timeval atime_mtime[2]; if (mtime) { - atime_mtime[0].tv_sec = atime ? atime->tv_sec : mtime->tv_sec; - atime_mtime[0].tv_usec = (atime ? atime->tv_nsec : mtime->tv_nsec) / 1000; - atime_mtime[1].tv_sec = mtime->tv_sec; - atime_mtime[1].tv_usec = mtime->tv_nsec / 1000; + atime_mtime[0].tv_sec = atime ? atime->sec() : mtime->sec(); + atime_mtime[0].tv_usec = + (atime ? atime->nsec_decimal_part() : mtime->nsec_decimal_part()) / 1000; + atime_mtime[1].tv_sec = mtime->sec(); + atime_mtime[1].tv_usec = mtime->nsec_decimal_part() / 1000; } utimes(path.c_str(), mtime ? atime_mtime : nullptr); #else utimbuf atime_mtime; if (mtime) { - atime_mtime.actime = atime ? atime->tv_sec : mtime->tv_sec; - atime_mtime.modtime = mtime->tv_sec; + atime_mtime.actime = atime ? atime->sec() : mtime->sec(); + atime_mtime.modtime = mtime->sec(); utime(path.c_str(), &atime_mtime); } else { utime(path.c_str(), nullptr); diff --git a/src/util/file.hpp b/src/util/file.hpp index 7ebdbdbec..bb0c8b226 100644 --- a/src/util/file.hpp +++ b/src/util/file.hpp @@ -18,6 +18,7 @@ #pragma once +#include #include #include @@ -67,8 +68,8 @@ read_file_part(const std::string& path, size_t pos, size_t count); // Set atime/mtime of `path`. If `mtime` is std::nullopt, set to the current // time. If `atime` is std::nullopt, set to what `mtime` specifies. void set_timestamps(const std::string& path, - std::optional mtime = std::nullopt, - std::optional atime = std::nullopt); + std::optional mtime = std::nullopt, + std::optional atime = std::nullopt); // Write `size` bytes from binary `data` to `fd`. nonstd::expected diff --git a/unittest/test_Stat.cpp b/unittest/test_Stat.cpp index b7e6178aa..fbd0afe5c 100644 --- a/unittest/test_Stat.cpp +++ b/unittest/test_Stat.cpp @@ -154,20 +154,16 @@ TEST_CASE("Default constructor") CHECK(stat.device() == 0); CHECK(stat.inode() == 0); CHECK(stat.mode() == 0); - CHECK(stat.ctime() == 0); - CHECK(stat.mtime() == 0); + CHECK(stat.ctime().sec() == 0); + CHECK(stat.ctime().nsec() == 0); + CHECK(stat.mtime().sec() == 0); + CHECK(stat.mtime().nsec() == 0); CHECK(stat.size() == 0); CHECK(stat.size_on_disk() == 0); CHECK(!stat.is_directory()); CHECK(!stat.is_regular()); CHECK(!stat.is_symlink()); - CHECK(stat.ctim().tv_sec == 0); - CHECK(stat.ctim().tv_nsec == 0); - - CHECK(stat.mtim().tv_sec == 0); - CHECK(stat.mtim().tv_nsec == 0); - #ifdef _WIN32 CHECK(stat.file_attributes() == 0); CHECK(stat.reparse_tag() == 0); @@ -210,20 +206,16 @@ TEST_CASE("Return values when file is missing") CHECK(stat.device() == 0); CHECK(stat.inode() == 0); CHECK(stat.mode() == 0); - CHECK(stat.ctime() == 0); - CHECK(stat.mtime() == 0); + CHECK(stat.ctime().sec() == 0); + CHECK(stat.ctime().nsec() == 0); + CHECK(stat.mtime().sec() == 0); + CHECK(stat.mtime().nsec() == 0); CHECK(stat.size() == 0); CHECK(stat.size_on_disk() == 0); CHECK(!stat.is_directory()); CHECK(!stat.is_regular()); CHECK(!stat.is_symlink()); - CHECK(stat.ctim().tv_sec == 0); - CHECK(stat.ctim().tv_nsec == 0); - - CHECK(stat.mtim().tv_sec == 0); - CHECK(stat.mtim().tv_nsec == 0); - #ifdef _WIN32 CHECK(stat.file_attributes() == 0); CHECK(stat.reparse_tag() == 0); @@ -259,13 +251,10 @@ TEST_CASE("Return values when file exists") struct timespec last_write_time = win32_filetime_to_timespec(info.ftLastWriteTime); - CHECK(stat.ctime() == creation_time.tv_sec); - CHECK(stat.mtime() == last_write_time.tv_sec); - - CHECK(stat.ctim().tv_sec == creation_time.tv_sec); - CHECK(stat.ctim().tv_nsec == creation_time.tv_nsec); - CHECK(stat.mtim().tv_sec == last_write_time.tv_sec); - CHECK(stat.mtim().tv_nsec == last_write_time.tv_nsec); + CHECK(stat.ctime().sec() == creation_time.tv_sec); + CHECK(stat.ctime().nsec_decimal_part() == creation_time.tv_nsec); + CHECK(stat.mtime().sec() == last_write_time.tv_sec); + CHECK(stat.mtime().nsec_decimal_part() == last_write_time.tv_nsec); CHECK(stat.size_on_disk() == ((stat.size() + 1023) & ~1023)); CHECK(stat.file_attributes() == info.dwFileAttributes); @@ -278,30 +267,28 @@ TEST_CASE("Return values when file exists") CHECK(stat.device() == st.st_dev); CHECK(stat.inode() == st.st_ino); CHECK(stat.mode() == st.st_mode); - CHECK(stat.ctime() == st.st_ctime); - CHECK(stat.mtime() == st.st_mtime); CHECK(stat.size_on_disk() == st.st_blocks * 512); # ifdef HAVE_STRUCT_STAT_ST_CTIM - CHECK(stat.ctim().tv_sec == st.st_ctim.tv_sec); - CHECK(stat.ctim().tv_nsec == st.st_ctim.tv_nsec); + CHECK(stat.ctime().sec() == st.st_ctim.tv_sec); + CHECK(stat.ctime().nsec_decimal_part() == st.st_ctim.tv_nsec); # elif defined(HAVE_STRUCT_STAT_ST_CTIMESPEC) - CHECK(stat.ctim().tv_sec == st.st_ctimespec.tv_sec); - CHECK(stat.ctim().tv_nsec == st.st_ctimespec.tv_nsec); + CHECK(stat.ctime().sec() == st.st_ctimespec.tv_sec); + CHECK(stat.ctime().nsec_decimal_part() == st.st_ctimespec.tv_nsec); # else - CHECK(stat.ctim().tv_sec == st.st_ctime); - CHECK(stat.ctim().tv_nsec == 0); + CHECK(stat.ctime().sec() == st.st_ctime); + CHECK(stat.ctime().nsec_decimal_part() == 0); # endif # ifdef HAVE_STRUCT_STAT_ST_MTIM - CHECK(stat.mtim().tv_sec == st.st_mtim.tv_sec); - CHECK(stat.mtim().tv_nsec == st.st_mtim.tv_nsec); + CHECK(stat.mtime().sec() == st.st_mtim.tv_sec); + CHECK(stat.mtime().nsec_decimal_part() == st.st_mtim.tv_nsec); # elif defined(HAVE_STRUCT_STAT_ST_MTIMESPEC) - CHECK(stat.mtim().tv_sec == st.st_mtimespec.tv_sec); - CHECK(stat.mtim().tv_nsec == st.st_mtimespec.tv_nsec); + CHECK(stat.mtime().sec() == st.st_mtimespec.tv_sec); + CHECK(stat.mtime().nsec_decimal_part() == st.st_mtimespec.tv_nsec); # else - CHECK(stat.mtim().tv_sec == st.st_mtime); - CHECK(stat.mtim().tv_nsec == 0); + CHECK(stat.mtime().sec() == st.st_mtime); + CHECK(stat.mtime().nsec_decimal_part() == 0); # endif #endif } diff --git a/unittest/test_Util.cpp b/unittest/test_Util.cpp index d16d71351..b71dfdb33 100644 --- a/unittest/test_Util.cpp +++ b/unittest/test_Util.cpp @@ -38,7 +38,6 @@ #include -using doctest::Approx; using TestUtil::TestContext; TEST_SUITE_BEGIN("Util"); diff --git a/unittest/test_util_LockFile.cpp b/unittest/test_util_LockFile.cpp index f8fe05db1..ff3153c66 100644 --- a/unittest/test_util_LockFile.cpp +++ b/unittest/test_util_LockFile.cpp @@ -136,7 +136,7 @@ TEST_CASE("Break stale lock, blocking") TestContext test_context; util::write_file("test.alive", ""); - const timespec long_time_ago{0, 0}; + const util::TimePoint long_time_ago(0, 0); util::set_timestamps("test.alive", long_time_ago); CHECK(symlink("foo", "test.lock") == 0); @@ -150,7 +150,7 @@ TEST_CASE("Break stale lock, non-blocking") TestContext test_context; util::write_file("test.alive", ""); - const timespec long_time_ago{0, 0}; + const util::TimePoint long_time_ago(0, 0); util::set_timestamps("test.alive", long_time_ago); CHECK(symlink("foo", "test.lock") == 0);