From: Olle Liljenzin Date: Sun, 31 May 2020 10:02:12 +0000 (+0200) Subject: Add inode cache for file hashes (#577) X-Git-Tag: v4.0~426 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=213d9883;p=thirdparty%2Fccache.git Add inode cache for file hashes (#577) The inode cache is a process shared cache that maps from device, inode, size, mtime and ctime to saved hash results. The cache is stored persistently in a single file that is mapped into shared memory by running processes, allowing computed hash values to be reused both within and between builds. The chosen technical solution works for Linux and might work on other POSIX platforms, but is not meant to be supported on non-POSIX platforms such as Windows. Use 'ccache -o inode_cache=true/false' to activate/deactivate the cache. --- diff --git a/Makefile.in b/Makefile.in index 8831ddebc..2b2bb9e4e 100644 --- a/Makefile.in +++ b/Makefile.in @@ -42,6 +42,7 @@ non_third_party_sources = \ src/Context.cpp \ src/Counters.cpp \ src/Decompressor.cpp \ + src/InodeCache.cpp \ src/Lockfile.cpp \ src/MiniTrace.cpp \ src/NullCompressor.cpp \ @@ -89,6 +90,7 @@ test_suites += unittest/test_Checksum.cpp test_suites += unittest/test_Compression.cpp test_suites += unittest/test_Config.cpp test_suites += unittest/test_FormatNonstdStringView.cpp +test_suites += unittest/test_InodeCache.cpp test_suites += unittest/test_Lockfile.cpp test_suites += unittest/test_NullCompression.cpp test_suites += unittest/test_Stat.cpp diff --git a/configure.ac b/configure.ac index a900ed626..33f6c7d28 100644 --- a/configure.ac +++ b/configure.ac @@ -120,11 +120,13 @@ AC_CHECK_HEADERS(linux/fs.h) AC_CHECK_HEADERS(sys/clonefile.h) AC_CHECK_FUNCS(asctime_r) +AC_CHECK_FUNCS(geteuid) AC_CHECK_FUNCS(getopt_long) AC_CHECK_FUNCS(getpwuid) AC_CHECK_FUNCS(gettimeofday) AC_CHECK_FUNCS(localtime_r) AC_CHECK_FUNCS(mkstemp) +AC_CHECK_FUNCS(posix_fallocate) AC_CHECK_FUNCS(realpath) AC_CHECK_FUNCS(setenv) AC_CHECK_FUNCS(strndup) @@ -132,6 +134,9 @@ AC_CHECK_FUNCS(syslog) AC_CHECK_FUNCS(unsetenv) AC_CHECK_FUNCS(utimes) +AC_CHECK_MEMBERS([struct stat.st_ctim, struct stat.st_mtim]) +AC_CHECK_MEMBERS([struct statfs.f_fstypename], [], [], [#include ]) + dnl Check if -lm is needed. AC_SEARCH_LIBS(cos, m) diff --git a/doc/MANUAL.adoc b/doc/MANUAL.adoc index 5b20df4a9..6abf19661 100644 --- a/doc/MANUAL.adoc +++ b/doc/MANUAL.adoc @@ -495,6 +495,17 @@ might be incorrect. change. The list separator is semicolon on Windows systems and colon on other systems. +[[config_inode_cache]] *inode_cache* (*CCACHE_INODECACHE* or *CCACHE_NOINODECACHE*, see <<_boolean_values,Boolean values>> above):: + + If true, enables caching of source file hashes based on device, inode and + timestamps. This will reduce the time spent on hashing included files as + the result can be resused between compilations. ++ +The feature is still experimental and thus off by default. It is currently not +available on Windows. ++ +The feature requires *temporary_dir* to be located on a local filesystem. + [[config_keep_comments_cpp]] *keep_comments_cpp* (*CCACHE_COMMENTS* or *CCACHE_NOCOMMENTS*, see <<_boolean_values,Boolean values>> above):: If true, ccache will not discard the comments before hashing preprocessor @@ -666,7 +677,8 @@ information. [[config_temporary_dir]] *temporary_dir* (*CCACHE_TEMPDIR*):: This setting specifies where ccache will put temporary files. The default - is */tmp*. + is */run/user//ccache-tmp* if */run/user/* exists, otherwise + */tmp*. + NOTE: In previous versions of ccache, *CCACHE_TEMPDIR* had to be on the same filesystem as the *CCACHE_DIR* path, but this requirement has been diff --git a/src/Config.cpp b/src/Config.cpp index da87c3d95..286cae50e 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -54,6 +54,7 @@ enum class ConfigItem { hard_link, hash_dir, ignore_headers_in_manifest, + inode_cache, keep_comments_cpp, limit_multiple, log_file, @@ -91,6 +92,7 @@ const std::unordered_map k_config_key_table = { {"hard_link", ConfigItem::hard_link}, {"hash_dir", ConfigItem::hash_dir}, {"ignore_headers_in_manifest", ConfigItem::ignore_headers_in_manifest}, + {"inode_cache", ConfigItem::inode_cache}, {"keep_comments_cpp", ConfigItem::keep_comments_cpp}, {"limit_multiple", ConfigItem::limit_multiple}, {"log_file", ConfigItem::log_file}, @@ -130,6 +132,7 @@ const std::unordered_map k_env_variable_table = { {"HARDLINK", "hard_link"}, {"HASHDIR", "hash_dir"}, {"IGNOREHEADERS", "ignore_headers_in_manifest"}, + {"INODECACHE", "inode_cache"}, {"LIMIT_MULTIPLE", "limit_multiple"}, {"LOGFILE", "log_file"}, {"MAXFILES", "max_files"}, @@ -556,6 +559,9 @@ Config::get_string_value(const std::string& key) const case ConfigItem::ignore_headers_in_manifest: return m_ignore_headers_in_manifest; + case ConfigItem::inode_cache: + return format_bool(m_inode_cache); + case ConfigItem::keep_comments_cpp: return format_bool(m_keep_comments_cpp); @@ -760,6 +766,10 @@ Config::set_item(const std::string& key, m_ignore_headers_in_manifest = parse_env_string(value); break; + case ConfigItem::inode_cache: + m_inode_cache = parse_bool(value, env_var_key, negate); + break; + case ConfigItem::keep_comments_cpp: m_keep_comments_cpp = parse_bool(value, env_var_key, negate); break; @@ -845,3 +855,14 @@ Config::check_key_tables_consistency() } } } + +std::string +Config::default_temporary_dir(const std::string& cache_dir) +{ +#ifdef HAVE_GETEUID + if (Stat::stat(fmt::format("/run/user/{}", geteuid())).is_directory()) { + return fmt::format("/run/user/{}/ccache-tmp", geteuid()); + } +#endif + return cache_dir + "/tmp"; +} diff --git a/src/Config.hpp b/src/Config.hpp index cde10497c..b7a3c80a1 100644 --- a/src/Config.hpp +++ b/src/Config.hpp @@ -55,6 +55,7 @@ public: bool hard_link() const; bool hash_dir() const; const std::string& ignore_headers_in_manifest() const; + bool inode_cache() const; bool keep_comments_cpp() const; double limit_multiple() const; const std::string& log_file() const; @@ -77,7 +78,9 @@ public: void set_cache_dir(const std::string& value); void set_cpp_extension(const std::string& value); void set_depend_mode(bool value); + void set_debug(bool value); void set_direct_mode(bool value); + void set_inode_cache(bool value); void set_limit_multiple(double value); void set_max_files(uint32_t value); void set_max_size(uint64_t value); @@ -139,6 +142,7 @@ private: bool m_hard_link = false; bool m_hash_dir = true; std::string m_ignore_headers_in_manifest = ""; + bool m_inode_cache = false; bool m_keep_comments_cpp = false; double m_limit_multiple = 0.8; std::string m_log_file = ""; @@ -154,7 +158,7 @@ private: bool m_run_second_cpp = true; uint32_t m_sloppiness = 0; bool m_stats = true; - std::string m_temporary_dir = fmt::format(m_cache_dir + "/tmp"); + std::string m_temporary_dir = default_temporary_dir(m_cache_dir); uint32_t m_umask = std::numeric_limits::max(); // Don't set umask bool m_temporary_dir_configured_explicitly = false; @@ -166,6 +170,8 @@ private: const nonstd::optional& env_var_key, bool negate, const std::string& origin); + + static std::string default_temporary_dir(const std::string& cache_dir); }; inline const std::string& @@ -270,6 +276,12 @@ Config::ignore_headers_in_manifest() const return m_ignore_headers_in_manifest; } +inline bool +Config::inode_cache() const +{ + return m_inode_cache; +} + inline bool Config::keep_comments_cpp() const { @@ -383,7 +395,7 @@ Config::set_cache_dir(const std::string& value) { m_cache_dir = value; if (!m_temporary_dir_configured_explicitly) { - m_temporary_dir = m_cache_dir + "/tmp"; + m_temporary_dir = default_temporary_dir(m_cache_dir); } } @@ -399,12 +411,24 @@ Config::set_depend_mode(bool value) m_depend_mode = value; } +inline void +Config::set_debug(bool value) +{ + m_debug = value; +} + inline void Config::set_direct_mode(bool value) { m_direct_mode = value; } +inline void +Config::set_inode_cache(bool value) +{ + m_inode_cache = value; +} + inline void Config::set_limit_multiple(double value) { diff --git a/src/Context.cpp b/src/Context.cpp index bcbefe9a4..ec380237c 100644 --- a/src/Context.cpp +++ b/src/Context.cpp @@ -30,6 +30,10 @@ using nonstd::string_view; Context::Context() : actual_cwd(Util::get_actual_cwd()), apparent_cwd(Util::get_apparent_cwd(actual_cwd)) +#ifdef INODE_CACHE_SUPPORTED + , + inode_cache(config) +#endif { } diff --git a/src/Context.hpp b/src/Context.hpp index b4b393835..246b978fd 100644 --- a/src/Context.hpp +++ b/src/Context.hpp @@ -24,6 +24,7 @@ #include "ArgsInfo.hpp" #include "Config.hpp" #include "File.hpp" +#include "InodeCache.hpp" #include "MiniTrace.hpp" #include "NonCopyable.hpp" #include "ccache.hpp" @@ -102,6 +103,11 @@ public: // Headers (or directories with headers) to ignore in manifest mode. std::vector ignore_header_paths; +#ifdef INODE_CACHE_SUPPORTED + // InodeCache that caches source file hashes when enabled. + mutable InodeCache inode_cache; +#endif + // Full path to the statistics file in the subdirectory where the cached // result belongs (//stats). const std::string& stats_file() const; diff --git a/src/InodeCache.cpp b/src/InodeCache.cpp new file mode 100644 index 000000000..46dcd8ad0 --- /dev/null +++ b/src/InodeCache.cpp @@ -0,0 +1,494 @@ +// Copyright (C) 2020 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "InodeCache.hpp" + +#ifdef INODE_CACHE_SUPPORTED + +# include "Config.hpp" +# include "Stat.hpp" +# include "Util.hpp" +# include "ccache.hpp" +# include "hash.hpp" +# include "logging.hpp" + +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include + +namespace { + +// The inode cache resides on a file that is mapped into shared memory by +// running processes. It is implemented as a two level structure, where the +// top level is a hash table consisting of buckets. Each bucket contains entries +// that are sorted in LRU order. Entries map from keys representing files to +// cached hash results. +// +// Concurrent access is guarded by a mutex in each bucket. +// +// Current cache size is fixed and the given constants are considered large +// enough for most projects. The size could be made configurable if there is a +// demand for it. +const uint32_t k_version = 1; + +// Increment version number if constants affecting storage size are changed. +const uint32_t k_num_buckets = 32 * 1024; +const uint32_t k_num_entries = 4; + +static_assert(sizeof(digest::bytes) == 20, + "Increment version number if size of digest is changed."); + +static_assert( + static_cast(InodeCache::ContentType::binary) == 0, + "Numeric value is part of key, increment version number if changed."); +static_assert( + static_cast(InodeCache::ContentType::code) == 1, + "Numeric value is part of key, increment version number if changed."); +static_assert( + static_cast(InodeCache::ContentType::code_with_sloppy_time_macros) == 2, + "Numeric value is part of key, increment version number if changed."); +static_assert( + static_cast(InodeCache::ContentType::precompiled_header) == 3, + "Numeric value is part of key, increment version number if changed."); + +} // namespace + +struct InodeCache::Key +{ + ContentType type; + dev_t st_dev; + ino_t st_ino; + mode_t st_mode; +# ifdef HAVE_STRUCT_STAT_ST_MTIM + timespec st_mtim; +# else + time_t st_mtim; +# endif +# ifdef HAVE_STRUCT_STAT_ST_CTIM + timespec st_ctim; // Included for sanity checking. +# else + time_t st_ctim; // Included for sanity checking. +# endif + off_t st_size; // Included for sanity checking. + bool sloppy_time_macros; +}; + +struct InodeCache::Entry +{ + digest key_digest; // Hashed key + digest file_digest; // Cached file hash + int return_value; // Cached return value +}; + +struct InodeCache::Bucket +{ + pthread_mutex_t mt; + Entry entries[k_num_entries]; +}; + +struct InodeCache::SharedRegion +{ + uint32_t version; + std::atomic hits; + std::atomic misses; + std::atomic errors; + Bucket buckets[k_num_buckets]; +}; + +bool +InodeCache::mmap_file(const std::string& inode_cache_file) +{ + if (m_sr) { + munmap(m_sr, sizeof(SharedRegion)); + m_sr = nullptr; + } + int fd = open(inode_cache_file.c_str(), O_RDWR); + if (fd < 0) { + cc_log("Failed to open inode cache %s: %s", + inode_cache_file.c_str(), + strerror(errno)); + return false; + } + bool is_nfs; + if (Util::is_nfs_fd(fd, &is_nfs) == 0 && is_nfs) { + cc_log( + "Inode cache not supported because the cache file is located on nfs: %s", + inode_cache_file.c_str()); + close(fd); + return false; + } + SharedRegion* sr = reinterpret_cast(mmap( + nullptr, sizeof(SharedRegion), PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0)); + close(fd); + if (sr == reinterpret_cast(-1)) { + cc_log("Failed to mmap %s: %s", inode_cache_file.c_str(), strerror(errno)); + return false; + } + // Drop the file from disk if the found version is not matching. This will + // allow a new file to be generated. + if (sr->version != k_version) { + cc_log( + "Dropping inode cache because found version %u does not match expected " + "version %u", + sr->version, + k_version); + munmap(sr, sizeof(SharedRegion)); + unlink(inode_cache_file.c_str()); + return false; + } + m_sr = sr; + if (m_config.debug()) { + cc_log("inode cache file loaded: %s", inode_cache_file.c_str()); + } + return true; +} + +bool +InodeCache::hash_inode(const char* path, ContentType type, digest* digest) +{ + Stat stat = Stat::stat(path); + if (!stat) { + cc_log("Could not stat %s: %s", path, strerror(stat.error_number())); + return false; + } + + Key key; + memset(&key, 0, sizeof(Key)); + key.type = type; + key.st_dev = stat.device(); + key.st_ino = stat.inode(); + key.st_mode = stat.mode(); +# ifdef HAVE_STRUCT_STAT_ST_MTIM + key.st_mtim = stat.mtim(); +# else + key.st_mtim = stat.mtime(); +# endif +# ifdef HAVE_STRUCT_STAT_ST_CTIM + key.st_ctim = stat.ctim(); +# else + key.st_ctim = stat.ctime(); +# endif + key.st_size = stat.size(); + + struct hash* hash = hash_init(); + hash_buffer(hash, &key, sizeof(Key)); + hash_result_as_bytes(hash, digest); + hash_free(hash); + return true; +} + +InodeCache::Bucket* +InodeCache::acquire_bucket(uint32_t index) +{ + Bucket* bucket = &m_sr->buckets[index]; + int err = pthread_mutex_lock(&bucket->mt); +# ifdef PTHREAD_MUTEX_ROBUST + if (err == EOWNERDEAD) { + if (m_config.debug()) + ++m_sr->errors; + err = pthread_mutex_consistent(&bucket->mt); + if (err) { + cc_log( + "Can't consolidate stale mutex at index %u: %s", index, strerror(err)); + cc_log("Consider removing the inode cache file if the problem persists"); + return nullptr; + } + cc_log("Wiping bucket at index %u because of stale mutex", index); + memset(bucket->entries, 0, sizeof(Bucket::entries)); + } else { +# endif + if (err) { + cc_log("Failed to lock mutex at index %u: %s", index, strerror(err)); + cc_log("Consider removing the inode cache file if problem persists"); + ++m_sr->errors; + return nullptr; + } +# ifdef PTHREAD_MUTEX_ROBUST + } +# endif + return bucket; +} + +InodeCache::Bucket* +InodeCache::acquire_bucket(const digest& key_digest) +{ + uint32_t hash; + Util::big_endian_to_int(key_digest.bytes, hash); + return acquire_bucket(hash % k_num_buckets); +} + +void +InodeCache::release_bucket(Bucket* bucket) +{ + pthread_mutex_unlock(&bucket->mt); +} + +bool +InodeCache::create_new_file(const std::string& filename) +{ + cc_log("Creating a new inode cache"); + + // Create the new file to a temporary name to prevent other processes from + // mapping it before it is fully initialized. + auto temp_fd = Util::create_temp_fd(filename); + bool is_nfs; + if (Util::is_nfs_fd(temp_fd.first, &is_nfs) == 0 && is_nfs) { + cc_log( + "Inode cache not supported because the cache file would be located on " + "nfs: %s", + filename.c_str()); + unlink(temp_fd.second.c_str()); + close(temp_fd.first); + return false; + } + int err = Util::fallocate(temp_fd.first, sizeof(SharedRegion)); + if (err) { + cc_log("Failed to allocate file space for inode cache: %s", strerror(err)); + unlink(temp_fd.second.c_str()); + close(temp_fd.first); + return false; + } + SharedRegion* sr = + reinterpret_cast(mmap(nullptr, + sizeof(SharedRegion), + PROT_READ | PROT_WRITE, + MAP_SHARED, + temp_fd.first, + 0)); + if (sr == reinterpret_cast(-1)) { + cc_log("Failed to mmap new inode cache: %s", strerror(errno)); + unlink(temp_fd.second.c_str()); + close(temp_fd.first); + return false; + } + + // Initialize new shared region. + sr->version = k_version; + pthread_mutexattr_t mattr; + pthread_mutexattr_init(&mattr); + pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED); +# ifdef PTHREAD_MUTEX_ROBUST + pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST); +# endif + for (uint32_t i = 0; i < k_num_buckets; ++i) { + pthread_mutex_init(&sr->buckets[i].mt, &mattr); + } + + munmap(sr, sizeof(SharedRegion)); + close(temp_fd.first); + + // link() will fail silently if a file with the same name already exists. + // This will be the case if two processes try to create a new file + // simultaneously. Thus close the current file handle and reopen a new one, + // which will make us use the first created file even if we didn't win the + // race. + if (link(temp_fd.second.c_str(), filename.c_str()) != 0) { + cc_log("Failed to link new inode cache: %s", strerror(errno)); + unlink(temp_fd.second.c_str()); + return false; + } + + unlink(temp_fd.second.c_str()); + return true; +} + +bool +InodeCache::initialize() +{ + if (m_failed || !m_config.inode_cache()) { + return false; + } + + if (m_sr) { + return true; + } + + std::string filename = get_file(); + if (m_sr || mmap_file(filename)) { + return true; + } + + // Try to create a new cache if we failed to map an existing file. + create_new_file(filename); + + // Concurrent processes could try to create new files simultaneously and the + // file that actually landed on disk will be from the process that won the + // race. Thus we try to open the file from disk instead of reusing the file + // handle to the file we just created. + if (mmap_file(filename)) { + return true; + } + + m_failed = true; + return false; +} + +InodeCache::InodeCache(const Config& config) + : m_config(config), m_sr(nullptr), m_failed(false) +{ +} + +InodeCache::~InodeCache() +{ + if (m_sr) { + munmap(m_sr, sizeof(SharedRegion)); + } +} + +bool +InodeCache::get(const char* path, + ContentType type, + digest* file_digest, + int* return_value) +{ + if (!initialize()) { + return false; + } + + digest key_digest; + if (!hash_inode(path, type, &key_digest)) { + return false; + } + + Bucket* bucket = acquire_bucket(key_digest); + + if (!bucket) { + return false; + } + + bool found = false; + + for (uint32_t i = 0; i < k_num_entries; ++i) { + if (digests_equal(&bucket->entries[i].key_digest, &key_digest)) { + if (i > 0) { + Entry tmp = bucket->entries[i]; + memmove(&bucket->entries[1], &bucket->entries[0], sizeof(Entry) * i); + bucket->entries[0] = tmp; + } + + *file_digest = bucket->entries[0].file_digest; + if (return_value) { + *return_value = bucket->entries[0].return_value; + } + found = true; + break; + } + } + release_bucket(bucket); + + cc_log("inode cache %s: %s", found ? "hit" : "miss", path); + + if (m_config.debug()) { + if (found) { + ++m_sr->hits; + } else { + ++m_sr->misses; + } + cc_log( + "accumulated stats for inode cache: hits=%ld, misses=%ld, errors=%ld", + static_cast(m_sr->hits.load()), + static_cast(m_sr->misses.load()), + static_cast(m_sr->errors.load())); + } + return found; +} + +bool +InodeCache::put(const char* path, + ContentType type, + const digest& file_digest, + int return_value) +{ + if (!initialize()) { + return false; + } + + digest key_digest; + if (!hash_inode(path, type, &key_digest)) { + return false; + } + + Bucket* bucket = acquire_bucket(key_digest); + + if (!bucket) { + return false; + } + + memmove(&bucket->entries[1], + &bucket->entries[0], + sizeof(Entry) * (k_num_entries - 1)); + + bucket->entries[0].key_digest = key_digest; + bucket->entries[0].file_digest = file_digest; + bucket->entries[0].return_value = return_value; + + release_bucket(bucket); + + cc_log("inode cache insert: %s", path); + + return true; +} + +bool +InodeCache::drop() +{ + std::string file = get_file(); + if (unlink(file.c_str()) != 0) { + return false; + } + if (m_sr) { + munmap(m_sr, sizeof(SharedRegion)); + m_sr = nullptr; + } + return true; +} + +std::string +InodeCache::get_file() +{ + return fmt::format("{}/inode-cache.v{}", m_config.temporary_dir(), k_version); +} + +int64_t +InodeCache::get_hits() +{ + return initialize() ? m_sr->hits.load() : -1; +} + +int64_t +InodeCache::get_misses() +{ + return initialize() ? m_sr->misses.load() : -1; +} + +int64_t +InodeCache::get_errors() +{ + return initialize() ? m_sr->errors.load() : -1; +} + +#endif diff --git a/src/InodeCache.hpp b/src/InodeCache.hpp new file mode 100644 index 000000000..3f45ca2f3 --- /dev/null +++ b/src/InodeCache.hpp @@ -0,0 +1,113 @@ +// Copyright (C) 2020 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include "system.hpp" + +#include "config.h" + +#ifdef INODE_CACHE_SUPPORTED + +# include +# include + +class Config; +class Context; +struct digest; + +class InodeCache +{ +public: + // Specifies in which role a file was hashed, since the hash result does not + // only depend on the actual content but also what we used it for. Source code + // files are scanned for macros while binary files are not as one example. + enum class ContentType { + binary = 0, + code = 1, + code_with_sloppy_time_macros = 2, + precompiled_header = 3, + }; + + InodeCache(const Config& config); + ~InodeCache(); + + // Get saved hash digest and return value from a previous call to + // hash_source_code_file(). + // + // Returns true if saved values could be retrieved from the cache, false + // otherwise. + bool get(const char* path, + ContentType type, + digest* file_digest, + int* return_value = nullptr); + + // Put hash digest and return value from a successful call to + // hash_source_code_file(). + // + // Returns true if values could be stored in the cache, false otherwise. + bool put(const char* path, + ContentType type, + const digest& file_digest, + int return_value = 0); + + // Unmaps the current cache and removes the mapped file from disk. + // + // Returns true on success, false otherwise. + bool drop(); + + // Returns name of the persistent file. + std::string get_file(); + + // Returns total number of cache hits. + // + // Counters are incremented in debug mode only. + int64_t get_hits(); + + // Returns total number of cache misses. + // + // Counters are incremented in debug mode only. + int64_t get_misses(); + + // Returns total number of errors. + // + // Currently only lock errors will be counted, since the counter is not + // accessible before the file has been successfully mapped into memory. + // + // Counters are incremented in debug mode only. + int64_t get_errors(); + +private: + struct Bucket; + struct Entry; + struct Key; + struct SharedRegion; + + bool mmap_file(const std::string& inode_cache_file); + bool hash_inode(const char* path, ContentType type, digest* digest); + Bucket* acquire_bucket(uint32_t index); + Bucket* acquire_bucket(const digest& key_digest); + void release_bucket(Bucket* bucket); + bool create_new_file(const std::string& filename); + bool initialize(); + + const Config& m_config; + struct SharedRegion* m_sr; + bool m_failed; +}; +#endif diff --git a/src/Stat.hpp b/src/Stat.hpp index 9f9fa1601..721b824a5 100644 --- a/src/Stat.hpp +++ b/src/Stat.hpp @@ -81,6 +81,14 @@ public: bool is_regular() const; bool is_symlink() const; +#ifdef HAVE_STRUCT_STAT_ST_CTIM + timespec ctim() const; +#endif + +#ifdef HAVE_STRUCT_STAT_ST_MTIM + timespec mtim() const; +#endif + protected: using StatFunction = int (*)(const char*, struct stat*); @@ -201,3 +209,19 @@ Stat::is_regular() const { return S_ISREG(mode()); } + +#ifdef HAVE_STRUCT_STAT_ST_CTIM +inline timespec +Stat::ctim() const +{ + return m_stat.st_ctim; +} +#endif + +#ifdef HAVE_STRUCT_STAT_ST_MTIM +inline timespec +Stat::mtim() const +{ + return m_stat.st_mtim; +} +#endif diff --git a/src/Util.cpp b/src/Util.cpp index d951be2d0..d69cc2919 100644 --- a/src/Util.cpp +++ b/src/Util.cpp @@ -21,8 +21,20 @@ #include "Config.hpp" #include "Context.hpp" #include "FormatNonstdStringView.hpp" +#include "legacy_util.hpp" #include "logging.hpp" +#include +#include + +#ifdef HAVE_LINUX_FS_H +# include +# include +#elif defined(HAVE_STRUCT_STATFS_F_FSTYPENAME) +# include +# include +#endif + #ifdef _WIN32 # include "win32compat.hpp" #endif @@ -187,6 +199,38 @@ ends_with(string_view string, string_view suffix) return string.ends_with(suffix); } +int +fallocate(int fd, long new_size) +{ +#ifdef HAVE_POSIX_FALLOCATE + return posix_fallocate(fd, 0, new_size); +#else + off_t saved_pos = lseek(fd, 0, SEEK_END); + off_t old_size = lseek(fd, 0, SEEK_END); + if (old_size == -1) { + int err = errno; + lseek(fd, saved_pos, SEEK_SET); + return err; + } + if (old_size >= new_size) { + lseek(fd, saved_pos, SEEK_SET); + return 0; + } + long bytes_to_write = new_size - old_size; + void* buf = calloc(bytes_to_write, 1); + if (!buf) { + lseek(fd, saved_pos, SEEK_SET); + return ENOMEM; + } + int err = 0; + if (!write_fd(fd, buf, bytes_to_write)) + err = errno; + lseek(fd, saved_pos, SEEK_SET); + free(buf); + return err; +#endif +} + void for_each_level_1_subdir(const std::string& cache_dir, const SubdirVisitor& subdir_visitor, @@ -386,6 +430,29 @@ is_absolute_path(string_view path) return !path.empty() && path[0] == '/'; } +#if defined(HAVE_LINUX_FS_H) || defined(HAVE_STRUCT_STATFS_F_FSTYPENAME) +int +is_nfs_fd(int fd, bool* is_nfs) +{ + struct statfs buf; + if (fstatfs(fd, &buf) != 0) { + return errno; + } +# ifdef HAVE_LINUX_FS_H + *is_nfs = buf.f_type == NFS_SUPER_MAGIC; +# else // Mac OS X and some other BSD flavors + *is_nfs = !strcmp(buf.f_fstypename, "nfs"); +# endif + return 0; +} +#else +int +is_nfs_fd([[gnu::unused]] int fd, [[gnu::unused]] bool* is_nfs) +{ + return -1; +} +#endif + std::string make_relative_path(const Context& ctx, string_view path) { diff --git a/src/Util.hpp b/src/Util.hpp index cdf4e1ea3..1f7871324 100644 --- a/src/Util.hpp +++ b/src/Util.hpp @@ -107,6 +107,15 @@ nonstd::string_view dir_name(nonstd::string_view path); // Return true if suffix is a suffix of string. bool ends_with(nonstd::string_view string, nonstd::string_view suffix); +// Extends file size to at least new_size by calling posix_fallocate() if +// supported, otherwise by writing zeros last to the file. +// +// Note that existing holes are not filled in case posix_fallocate() is not +// supported. +// +// Returns 0 on success, an error number otherwise. +int fallocate(int fd, long new_size); + // Call a function for each subdir (0-9a-f) in the cache. // // Parameters: @@ -209,6 +218,14 @@ int_to_big_endian(int8_t value, uint8_t* buffer) // Return whether `path` is absolute. bool is_absolute_path(nonstd::string_view path); +// Test if a file is on nfs. +// +// Sets is_nfs to the result if fstatfs is available and no error occurred. +// +// Returns 0 if is_nfs was set, -1 if fstatfs is not available or errno if an +// error occured. +int is_nfs_fd(int fd, bool* is_nfs); + // Return whether `ch` is a directory separator, i.e. '/' on POSIX systems and // '/' or '\\' on Windows systems. inline bool diff --git a/src/ccache.cpp b/src/ccache.cpp index eb15808c5..c24129620 100644 --- a/src/ccache.cpp +++ b/src/ccache.cpp @@ -342,7 +342,7 @@ do_remember_include_file(Context& ctx, } } - if (!hash_file(fhash, path.c_str())) { + if (!hash_binary_file(ctx, fhash, path.c_str())) { return false; } hash_delimiter(cpp_hash, using_pch_sum ? "pch_sum_hash" : "pch_hash"); @@ -353,20 +353,29 @@ do_remember_include_file(Context& ctx, if (ctx.config.direct_mode()) { if (!is_pch) { // else: the file has already been hashed. - char* source = nullptr; - size_t size; - if (st.size() > 0) { - if (!read_file(path.c_str(), st.size(), &source, &size)) { - return false; - } + int result; +#ifdef INODE_CACHE_SUPPORTED + if (ctx.config.inode_cache()) { + result = hash_source_code_file(ctx, fhash, path.c_str()); } else { - source = x_strdup(""); - size = 0; - } +#endif + char* source = nullptr; + size_t size; + if (st.size() > 0) { + if (!read_file(path.c_str(), st.size(), &source, &size)) { + return false; + } + } else { + source = x_strdup(""); + size = 0; + } - int result = - hash_source_code_string(ctx.config, fhash, source, size, path.c_str()); - free(source); + result = + hash_source_code_string(ctx, fhash, source, size, path.c_str()); + free(source); +#ifdef INODE_CACHE_SUPPORTED + } +#endif if (result & HASH_SOURCE_CODE_ERROR || result & HASH_SOURCE_CODE_FOUND_TIME) { return false; @@ -1075,7 +1084,8 @@ get_result_name_from_cpp(Context& ctx, Args& args, struct hash* hash) } hash_delimiter(hash, "cppstderr"); - if (!ctx.args_info.direct_i_file && !hash_file(hash, stderr_path.c_str())) { + if (!ctx.args_info.direct_i_file + && !hash_binary_file(ctx, hash, stderr_path.c_str())) { // Somebody removed the temporary file? cc_log("Failed to open %s: %s", stderr_path.c_str(), strerror(errno)); failed(STATS_ERROR); @@ -1125,7 +1135,7 @@ hash_compiler(const Context& ctx, hash_string(hash, ctx.config.compiler_check().c_str() + strlen("string:")); } else if (ctx.config.compiler_check() == "content" || !allow_command) { hash_delimiter(hash, "cc_content"); - hash_file(hash, path); + hash_binary_file(ctx, hash, path); } else { // command string if (!hash_multicommand_output(hash, ctx.config.compiler_check().c_str(), @@ -1296,7 +1306,7 @@ hash_common_info(const Context& ctx, for (const auto& sanitize_blacklist : args_info.sanitize_blacklists) { cc_log("Hashing sanitize blacklist %s", sanitize_blacklist.c_str()); hash_delimiter(hash, "sanitizeblacklist"); - if (!hash_file(hash, sanitize_blacklist.c_str())) { + if (!hash_binary_file(ctx, hash, sanitize_blacklist.c_str())) { failed(STATS_BADEXTRAFILE); } } @@ -1306,7 +1316,7 @@ hash_common_info(const Context& ctx, ctx.config.extra_files_to_hash(), PATH_DELIM)) { cc_log("Hashing extra file %s", path.c_str()); hash_delimiter(hash, "extrafile"); - if (!hash_file(hash, path.c_str())) { + if (!hash_binary_file(ctx, hash, path.c_str())) { failed(STATS_BADEXTRAFILE); } } @@ -1350,7 +1360,7 @@ hash_profile_data_file(const Context& ctx, struct hash* hash) if (st && !st.is_directory()) { cc_log("Adding profile data %s to the hash", p.c_str()); hash_delimiter(hash, "-fprofile-use"); - if (hash_file(hash, p.c_str())) { + if (hash_binary_file(ctx, hash, p.c_str())) { found = true; } } @@ -1602,7 +1612,7 @@ calculate_result_name(Context& ctx, hash_delimiter(hash, "sourcecode"); int result = - hash_source_code_file(ctx.config, hash, ctx.args_info.input_file.c_str()); + hash_source_code_file(ctx, hash, ctx.args_info.input_file.c_str()); if (result & HASH_SOURCE_CODE_ERROR) { failed(STATS_ERROR); } @@ -2256,7 +2266,7 @@ handle_main_options(int argc, const char* const* argv) if (str_eq(optarg, "-")) { hash_fd(hash, STDIN_FILENO); } else { - hash_file(hash, optarg); + hash_binary_file(ctx, hash, optarg); } char digest[DIGEST_STRING_BUFFER_SIZE]; hash_result_as_string(hash, digest); @@ -2283,8 +2293,7 @@ handle_main_options(int argc, const char* const* argv) case 'C': // --clear { ProgressBar progress_bar("Clearing..."); - wipe_all(ctx.config, - [&](double progress) { progress_bar.update(progress); }); + wipe_all(ctx, [&](double progress) { progress_bar.update(progress); }); if (isatty(STDOUT_FILENO)) { printf("\n"); } @@ -2345,7 +2354,7 @@ handle_main_options(int argc, const char* const* argv) break; case 's': // --show-stats - stats_summary(ctx.config); + stats_summary(ctx); break; case 'V': // --version @@ -2382,7 +2391,7 @@ handle_main_options(int argc, const char* const* argv) } case 'z': // --zero-stats - stats_zero(ctx.config); + stats_zero(ctx); printf("Statistics zeroed\n"); break; diff --git a/src/cleanup.cpp b/src/cleanup.cpp index 57e68c8ac..28b59fa42 100644 --- a/src/cleanup.cpp +++ b/src/cleanup.cpp @@ -21,6 +21,8 @@ #include "CacheFile.hpp" #include "Config.hpp" +#include "Context.hpp" +#include "InodeCache.hpp" #include "Util.hpp" #include "logging.hpp" #include "stats.hpp" @@ -189,8 +191,11 @@ wipe_dir(const std::string& subdir, // Wipe all cached files in all subdirectories. void -wipe_all(const Config& config, const Util::ProgressReceiver& progress_receiver) +wipe_all(const Context& ctx, const Util::ProgressReceiver& progress_receiver) { Util::for_each_level_1_subdir( - config.cache_dir(), wipe_dir, progress_receiver); + ctx.config.cache_dir(), wipe_dir, progress_receiver); +#ifdef INODE_CACHE_SUPPORTED + ctx.inode_cache.drop(); +#endif } diff --git a/src/cleanup.hpp b/src/cleanup.hpp index 4951fef32..1e34e66d4 100644 --- a/src/cleanup.hpp +++ b/src/cleanup.hpp @@ -25,6 +25,7 @@ #include class Config; +class Context; void clean_up_dir(const std::string& subdir, uint64_t max_size, @@ -34,5 +35,5 @@ void clean_up_dir(const std::string& subdir, void clean_up_all(const Config& config, const Util::ProgressReceiver& progress_receiver); -void wipe_all(const Config& config, +void wipe_all(const Context& ctx, const Util::ProgressReceiver& progress_receiver); diff --git a/src/hashutil.cpp b/src/hashutil.cpp index 246d5236f..e18e93ed2 100644 --- a/src/hashutil.cpp +++ b/src/hashutil.cpp @@ -21,6 +21,7 @@ #include "Args.hpp" #include "Config.hpp" #include "Context.hpp" +#include "InodeCache.hpp" #include "Stat.hpp" #include "ccache.hpp" #include "execute.hpp" @@ -188,7 +189,7 @@ check_for_temporal_macros(const char* str, size_t len) // Hash a string. Returns a bitmask of HASH_SOURCE_CODE_* results. int -hash_source_code_string(const Config& config, +hash_source_code_string(const Context& ctx, struct hash* hash, const char* str, size_t len, @@ -198,7 +199,7 @@ hash_source_code_string(const Config& config, // Check for __DATE__, __TIME__ and __TIMESTAMP__if the sloppiness // configuration tells us we should. - if (!(config.sloppiness() & SLOPPY_TIME_MACROS)) { + if (!(ctx.config.sloppiness() & SLOPPY_TIME_MACROS)) { result |= check_for_temporal_macros(str, len); } @@ -261,15 +262,14 @@ hash_source_code_string(const Config& config, return result; } -// Hash a file ignoring comments. Returns a bitmask of HASH_SOURCE_CODE_* -// results. -int -hash_source_code_file(const Config& config, - struct hash* hash, - const char* path, - size_t size_hint) +static int +hash_source_code_file_nocache(const Context& ctx, + struct hash* hash, + const char* path, + size_t size_hint, + bool is_precompiled) { - if (is_precompiled_header(path)) { + if (is_precompiled) { if (hash_file(hash, path)) { return HASH_SOURCE_CODE_OK; } else { @@ -281,12 +281,100 @@ hash_source_code_file(const Config& config, if (!read_file(path, size_hint, &data, &size)) { return HASH_SOURCE_CODE_ERROR; } - int result = hash_source_code_string(config, hash, data, size, path); + int result = hash_source_code_string(ctx, hash, data, size, path); free(data); return result; } } +#ifdef INODE_CACHE_SUPPORTED +static InodeCache::ContentType +get_content_type(const Config& config, const char* path) +{ + if (is_precompiled_header(path)) { + return InodeCache::ContentType::precompiled_header; + } + if (config.sloppiness() & SLOPPY_TIME_MACROS) { + return InodeCache::ContentType::code_with_sloppy_time_macros; + } + return InodeCache::ContentType::code; +} +#endif + +// Hash a file ignoring comments. Returns a bitmask of HASH_SOURCE_CODE_* +// results. +int +hash_source_code_file(const Context& ctx, + struct hash* hash, + const char* path, + size_t size_hint) +{ +#ifdef INODE_CACHE_SUPPORTED + if (!ctx.config.inode_cache()) { +#endif + return hash_source_code_file_nocache( + ctx, hash, path, size_hint, is_precompiled_header(path)); + +#ifdef INODE_CACHE_SUPPORTED + } + + // Reusable file hashes must be independent of the outer context. Thus hash + // files separately so that digests based on file contents can be reused. Then + // add the digest into the outer hash instead. + InodeCache::ContentType content_type = get_content_type(ctx.config, path); + struct digest digest; + int return_value; + if (!ctx.inode_cache.get(path, content_type, &digest, &return_value)) { + struct hash* file_hash = hash_init(); + return_value = hash_source_code_file_nocache( + ctx, + file_hash, + path, + size_hint, + content_type == InodeCache::ContentType::precompiled_header); + if (return_value == HASH_SOURCE_CODE_ERROR) { + return HASH_SOURCE_CODE_ERROR; + } + hash_result_as_bytes(file_hash, &digest); + hash_free(file_hash); + ctx.inode_cache.put(path, content_type, digest, return_value); + } + hash_buffer(hash, &digest.bytes, sizeof(digest::bytes)); + return return_value; +#endif +} + +// Hash a binary file using the inode cache if enabled. +// +// Returns true on success, otherwise false. +bool +hash_binary_file(const Context& ctx, struct hash* hash, const char* path) +{ + if (!ctx.config.inode_cache()) { + return hash_file(hash, path); + } + +#ifdef INODE_CACHE_SUPPORTED + // Reusable file hashes must be independent of the outer context. Thus hash + // files separately so that digests based on file contents can be reused. Then + // add the digest into the outer hash instead. + struct digest digest; + if (!ctx.inode_cache.get(path, InodeCache::ContentType::binary, &digest)) { + struct hash* file_hash = hash_init(); + if (!hash_file(hash, path)) { + return false; + } + hash_result_as_bytes(file_hash, &digest); + hash_free(file_hash); + ctx.inode_cache.put(path, InodeCache::ContentType::binary, digest); + } + hash_buffer(hash, &digest.bytes, sizeof(digest::bytes)); + return true; +#else + return hash_file(hash, path); +#endif +} + bool hash_command_output(struct hash* hash, const char* command, diff --git a/src/hashutil.hpp b/src/hashutil.hpp index d691538ca..5350b9a0e 100644 --- a/src/hashutil.hpp +++ b/src/hashutil.hpp @@ -36,15 +36,16 @@ unsigned hash_from_int(int i); #define HASH_SOURCE_CODE_FOUND_TIMESTAMP 8 int check_for_temporal_macros(const char* str, size_t len); -int hash_source_code_string(const Config& config, +int hash_source_code_string(const Context& ctx, struct hash* hash, const char* str, size_t len, const char* path); -int hash_source_code_file(const Config& config, +int hash_source_code_file(const Context& ctx, struct hash* hash, const char* path, size_t size_hint = 0); +bool hash_binary_file(const Context& ctx, struct hash* hash, const char* path); bool hash_command_output(struct hash* hash, const char* command, const char* compiler); diff --git a/src/manifest.cpp b/src/manifest.cpp index df2fb00cf..7399bc249 100644 --- a/src/manifest.cpp +++ b/src/manifest.cpp @@ -447,7 +447,7 @@ verify_result(const Context& ctx, auto hashed_files_iter = hashed_files.find(path); if (hashed_files_iter == hashed_files.end()) { struct hash* hash = hash_init(); - int ret = hash_source_code_file(ctx.config, hash, path.c_str(), fs.size); + int ret = hash_source_code_file(ctx, hash, path.c_str(), fs.size); if (ret & HASH_SOURCE_CODE_ERROR) { cc_log("Failed hashing %s", path.c_str()); hash_free(hash); diff --git a/src/stats.cpp b/src/stats.cpp index 5ebc8e395..5fc107e48 100644 --- a/src/stats.cpp +++ b/src/stats.cpp @@ -400,17 +400,18 @@ stats_update(Context& ctx, enum stats stat) // Sum and display the total stats for all cache dirs. void -stats_summary(const Config& config) +stats_summary(const Context& ctx) { Counters counters; time_t last_updated; - stats_collect(config, counters, &last_updated); + stats_collect(ctx.config, counters, &last_updated); - fmt::print("cache directory {}\n", config.cache_dir()); + fmt::print("cache directory {}\n", + ctx.config.cache_dir()); fmt::print("primary config {}\n", - config.primary_config_path()); + ctx.config.primary_config_path()); fmt::print("secondary config (readonly) {}\n", - config.secondary_config_path()); + ctx.config.secondary_config_path()); if (last_updated > 0) { struct tm tm; localtime_r(&last_updated, &tm); @@ -447,11 +448,11 @@ stats_summary(const Config& config) } } - if (config.max_files() != 0) { - printf("max files %8u\n", config.max_files()); + if (ctx.config.max_files() != 0) { + printf("max files %8u\n", ctx.config.max_files()); } - if (config.max_size() != 0) { - char* value = format_size(config.max_size()); + if (ctx.config.max_size() != 0) { + char* value = format_size(ctx.config.max_size()); printf("max cache size %s\n", value); free(value); } @@ -476,9 +477,9 @@ stats_print(const Config& config) // Zero all the stats structures. void -stats_zero(const Config& config) +stats_zero(const Context& ctx) { - char* fname = format("%s/stats", config.cache_dir().c_str()); + char* fname = format("%s/stats", ctx.config.cache_dir().c_str()); Util::unlink_safe(fname); free(fname); @@ -486,7 +487,7 @@ stats_zero(const Config& config) for (int dir = 0; dir <= 0xF; dir++) { Counters counters; - fname = format("%s/%1x/stats", config.cache_dir().c_str(), dir); + fname = format("%s/%1x/stats", ctx.config.cache_dir().c_str(), dir); if (!Stat::stat(fname)) { // No point in trying to reset the stats file if it doesn't exist. free(fname); diff --git a/src/stats.hpp b/src/stats.hpp index 4b3f7e30b..b67d20a65 100644 --- a/src/stats.hpp +++ b/src/stats.hpp @@ -71,8 +71,8 @@ void stats_flush(void* context); void stats_flush_to_file(const Config& config, std::string sfile, const Counters& updates); -void stats_zero(const Config& config); -void stats_summary(const Config& config); +void stats_zero(const Context& ctx); +void stats_summary(const Context& ctx); void stats_print(const Config& config); void stats_update_size(Counters& counters, int64_t size, int files); diff --git a/src/system.hpp b/src/system.hpp index 0b71cb097..f2c6fb3b5 100644 --- a/src/system.hpp +++ b/src/system.hpp @@ -111,3 +111,7 @@ extern char** environ; #ifndef O_BINARY # define O_BINARY 0 #endif + +#ifdef HAVE_SYS_MMAN_H +# define INODE_CACHE_SUPPORTED +#endif diff --git a/test/run b/test/run index 7b356d7b2..898b605e7 100755 --- a/test/run +++ b/test/run @@ -449,6 +449,7 @@ nvcc nvcc_direct nvcc_ldir nvcc_nocpp2 +inode_cache " for suite in $all_suites; do diff --git a/test/suites/base.bash b/test/suites/base.bash index e7ae45231..de5407284 100644 --- a/test/suites/base.bash +++ b/test/suites/base.bash @@ -540,7 +540,11 @@ base_tests() { # - a/b/c # - a/b/c/d actual_dirs=$(find $CCACHE_DIR -type d | wc -l) - expected_dirs=6 + if [ -d /run/user/$(id -u) ]; then + expected_dirs=5 + else + expected_dirs=6 + fi if [ $actual_dirs -ne $expected_dirs ]; then test_failed "Expected $expected_dirs directories, found $actual_dirs" fi diff --git a/test/suites/inode_cache.bash b/test/suites/inode_cache.bash new file mode 100644 index 000000000..bd38c454f --- /dev/null +++ b/test/suites/inode_cache.bash @@ -0,0 +1,81 @@ +SUITE_inode_cache_SETUP() { + export CCACHE_INODECACHE=1 + unset CCACHE_NODIRECT + cat /dev/null > $CCACHE_LOGFILE +} + +SUITE_inode_cache() { + inode_cache_tests +} + +expect_inode_cache_type() { + local expected=$1 + local source_file=$2 + local type=$3 + local actual=`grep "inode cache $type: $source_file" $CCACHE_LOGFILE|wc -l` + if [ $actual -ne $expected ]; then + test_failed "Found $actual (expected $expected) $type for $source_file" + fi +} + +expect_inode_cache() { + expect_inode_cache_type $1 $4 hit + expect_inode_cache_type $2 $4 miss + expect_inode_cache_type $3 $4 insert +} + +inode_cache_tests() { + # ------------------------------------------------------------------------- + TEST "Compile once" + + echo "// compile once" > test1.c + $CCACHE_COMPILE -c test1.c + expect_inode_cache 0 1 1 test1.c + + # ------------------------------------------------------------------------- + TEST "Recompile" + + echo "// recompile" > test1.c + $CCACHE_COMPILE -c test1.c + $CCACHE_COMPILE -c test1.c + expect_inode_cache 1 1 1 test1.c + + # ------------------------------------------------------------------------- + TEST "Backdate" + + echo "// backdate" > test1.c + $CCACHE_COMPILE -c test1.c + backdate test1.c + $CCACHE_COMPILE -c test1.c + expect_inode_cache 0 2 2 test1.c + + # ------------------------------------------------------------------------- + TEST "Hard link" + + echo "// hard linked" > test1.c + ln -f test1.c test2.c + $CCACHE_COMPILE -c test1.c + $CCACHE_COMPILE -c test2.c + expect_inode_cache 0 1 1 test1.c + expect_inode_cache 1 0 0 test2.c + + # ------------------------------------------------------------------------- + TEST "Soft link" + + echo "// soft linked" > test1.c + ln -fs test1.c test2.c + $CCACHE_COMPILE -c test1.c + $CCACHE_COMPILE -c test2.c + expect_inode_cache 0 1 1 test1.c + expect_inode_cache 1 0 0 test2.c + + # ------------------------------------------------------------------------- + TEST "Replace" + + echo "// replace" > test1.c + $CCACHE_COMPILE -c test1.c + rm test1.c + echo "// replace" > test1.c + $CCACHE_COMPILE -c test1.c + expect_inode_cache 0 2 2 test1.c +} diff --git a/unittest/test_Config.cpp b/unittest/test_Config.cpp index 7772ecd51..2ad3b0ded 100644 --- a/unittest/test_Config.cpp +++ b/unittest/test_Config.cpp @@ -71,7 +71,16 @@ TEST_CASE("Config: default values") CHECK(config.run_second_cpp()); CHECK(config.sloppiness() == 0); CHECK(config.stats()); - CHECK(config.temporary_dir() == expected_cache_dir + "/tmp"); +#ifdef HAVE_GETEUID + if (Stat::stat(fmt::format("/run/user/{}", geteuid())).is_directory()) { + CHECK(config.temporary_dir() + == fmt::format("/run/user/{}/ccache-tmp", geteuid())); + } else { +#endif + CHECK(config.temporary_dir() == expected_cache_dir + "/tmp"); +#ifdef HAVE_GETEUID + } +#endif CHECK(config.umask() == std::numeric_limits::max()); } @@ -394,6 +403,7 @@ TEST_CASE("Config::visit_items") "hard_link = true\n" "hash_dir = false\n" "ignore_headers_in_manifest = ihim\n" + "inode_cache = false\n" "keep_comments_cpp = true\n" "limit_multiple = 0.0\n" "log_file = lf\n" @@ -447,6 +457,7 @@ TEST_CASE("Config::visit_items") "(test.conf) hard_link = true", "(test.conf) hash_dir = false", "(test.conf) ignore_headers_in_manifest = ihim", + "(test.conf) inode_cache = false", "(test.conf) keep_comments_cpp = true", "(test.conf) limit_multiple = 0.0", "(test.conf) log_file = lf", diff --git a/unittest/test_InodeCache.cpp b/unittest/test_InodeCache.cpp new file mode 100644 index 000000000..8323aa38e --- /dev/null +++ b/unittest/test_InodeCache.cpp @@ -0,0 +1,208 @@ +// Copyright (C) 2020 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "../src/Config.hpp" +#include "../src/Context.hpp" +#include "../src/InodeCache.hpp" +#include "../src/Util.hpp" +#include "../src/hash.hpp" +#include "TestUtil.hpp" + +#include "third_party/catch.hpp" + +#ifdef INODE_CACHE_SUPPORTED + +using TestUtil::TestContext; + +namespace { + +struct digest +digest_from_string(const char* s) +{ + struct digest digest; + struct hash* hash = hash_init(); + hash_string(hash, s); + hash_result_as_bytes(hash, &digest); + hash_free(hash); + return digest; +} + +bool +digest_equals_string(const struct digest& digest, const char* s) +{ + struct digest rhs = digest_from_string(s); + return digests_equal(&digest, &rhs); +} + +bool +put(const Context& ctx, const char* filename, const char* s, int return_value) +{ + return ctx.inode_cache.put(filename, + InodeCache::ContentType::code, + digest_from_string(s), + return_value); +} + +} // namespace + +TEST_CASE("Test disabled") +{ + TestContext test_context; + + Context ctx; + ctx.config.set_debug(true); + ctx.config.set_inode_cache(false); + + struct digest digest; + int return_value; + + CHECK(!ctx.inode_cache.get( + "a", InodeCache::ContentType::code, &digest, &return_value)); + CHECK(!ctx.inode_cache.put( + "a", InodeCache::ContentType::code, digest, return_value)); + CHECK(ctx.inode_cache.get_hits() == -1); + CHECK(ctx.inode_cache.get_misses() == -1); + CHECK(ctx.inode_cache.get_errors() == -1); +} + +TEST_CASE("Test lookup nonexistent") +{ + TestContext test_context; + + Context ctx; + ctx.config.set_debug(true); + ctx.config.set_inode_cache(true); + ctx.inode_cache.drop(); + Util::write_file("a", ""); + + struct digest digest; + int return_value; + + CHECK(!ctx.inode_cache.get( + "a", InodeCache::ContentType::code, &digest, &return_value)); + CHECK(ctx.inode_cache.get_hits() == 0); + CHECK(ctx.inode_cache.get_misses() == 1); + CHECK(ctx.inode_cache.get_errors() == 0); +} + +TEST_CASE("Test put and lookup") +{ + TestContext test_context; + + Context ctx; + ctx.config.set_debug(true); + ctx.config.set_inode_cache(true); + ctx.inode_cache.drop(); + Util::write_file("a", "a text"); + + CHECK(put(ctx, "a", "a text", 1)); + + struct digest digest; + int return_value; + + CHECK(ctx.inode_cache.get( + "a", InodeCache::ContentType::code, &digest, &return_value)); + CHECK(digest_equals_string(digest, "a text")); + CHECK(return_value == 1); + CHECK(ctx.inode_cache.get_hits() == 1); + CHECK(ctx.inode_cache.get_misses() == 0); + CHECK(ctx.inode_cache.get_errors() == 0); + + Util::write_file("a", "something else"); + + CHECK(!ctx.inode_cache.get( + "a", InodeCache::ContentType::code, &digest, &return_value)); + CHECK(ctx.inode_cache.get_hits() == 1); + CHECK(ctx.inode_cache.get_misses() == 1); + CHECK(ctx.inode_cache.get_errors() == 0); + + CHECK(put(ctx, "a", "something else", 2)); + + CHECK(ctx.inode_cache.get( + "a", InodeCache::ContentType::code, &digest, &return_value)); + CHECK(digest_equals_string(digest, "something else")); + CHECK(return_value == 2); + CHECK(ctx.inode_cache.get_hits() == 2); + CHECK(ctx.inode_cache.get_misses() == 1); + CHECK(ctx.inode_cache.get_errors() == 0); +} + +TEST_CASE("Drop file") +{ + TestContext test_context; + + Context ctx; + ctx.config.set_debug(true); + ctx.config.set_inode_cache(true); + + struct digest digest; + + ctx.inode_cache.get("a", InodeCache::ContentType::binary, &digest); + CHECK(Stat::stat(ctx.inode_cache.get_file())); + CHECK(ctx.inode_cache.drop()); + CHECK(!Stat::stat(ctx.inode_cache.get_file())); + CHECK(!ctx.inode_cache.drop()); +} + +TEST_CASE("Test content type") +{ + TestContext test_context; + + Context ctx; + ctx.config.set_debug(true); + ctx.inode_cache.drop(); + ctx.config.set_inode_cache(true); + Util::write_file("a", "a text"); + digest binary_digest = digest_from_string("binary"); + digest code_digest = digest_from_string("code"); + digest code_with_sloppy_time_macros_digest = + digest_from_string("sloppy_time_macros"); + + CHECK(ctx.inode_cache.put( + "a", InodeCache::ContentType::binary, binary_digest, 1)); + CHECK( + ctx.inode_cache.put("a", InodeCache::ContentType::code, code_digest, 2)); + CHECK( + ctx.inode_cache.put("a", + InodeCache::ContentType::code_with_sloppy_time_macros, + code_with_sloppy_time_macros_digest, + 3)); + + digest digest; + int return_value; + + CHECK(ctx.inode_cache.get( + "a", InodeCache::ContentType::binary, &digest, &return_value)); + CHECK(digests_equal(&digest, &binary_digest)); + CHECK(return_value == 1); + + CHECK(ctx.inode_cache.get( + "a", InodeCache::ContentType::code, &digest, &return_value)); + CHECK(digests_equal(&digest, &code_digest)); + CHECK(return_value == 2); + + CHECK( + ctx.inode_cache.get("a", + InodeCache::ContentType::code_with_sloppy_time_macros, + &digest, + &return_value)); + CHECK(digests_equal(&digest, &code_with_sloppy_time_macros_digest)); + CHECK(return_value == 3); +} + +#endif diff --git a/unittest/test_Stat.cpp b/unittest/test_Stat.cpp index 8054e62d3..e5e424313 100644 --- a/unittest/test_Stat.cpp +++ b/unittest/test_Stat.cpp @@ -42,6 +42,16 @@ TEST_CASE("Default constructor") CHECK(!stat.is_directory()); CHECK(!stat.is_regular()); CHECK(!stat.is_symlink()); + +#ifdef HAVE_STRUCT_STAT_ST_CTIM + CHECK(stat.ctim().tv_sec == 0); + CHECK(stat.ctim().tv_nsec == 0); +#endif + +#ifdef HAVE_STRUCT_STAT_ST_MTIM + CHECK(stat.mtim().tv_sec == 0); + CHECK(stat.mtim().tv_nsec == 0); +#endif } TEST_CASE("Named constructors") @@ -90,6 +100,16 @@ TEST_CASE("Return values when file is missing") CHECK(!stat.is_directory()); CHECK(!stat.is_regular()); CHECK(!stat.is_symlink()); + +#ifdef HAVE_STRUCT_STAT_ST_CTIM + CHECK(stat.ctim().tv_sec == 0); + CHECK(stat.ctim().tv_nsec == 0); +#endif + +#ifdef HAVE_STRUCT_STAT_ST_MTIM + CHECK(stat.mtim().tv_sec == 0); + CHECK(stat.mtim().tv_nsec == 0); +#endif } TEST_CASE("Return values when file exists") @@ -118,6 +138,16 @@ TEST_CASE("Return values when file exists") CHECK(!stat.is_directory()); CHECK(stat.is_regular()); CHECK(!stat.is_symlink()); + +#ifdef HAVE_STRUCT_STAT_ST_CTIM + CHECK(stat.ctim().tv_sec == st.st_ctim.tv_sec); + CHECK(stat.ctim().tv_nsec == st.st_ctim.tv_nsec); +#endif + +#ifdef HAVE_STRUCT_STAT_ST_MTIM + CHECK(stat.mtim().tv_sec == st.st_mtim.tv_sec); + CHECK(stat.mtim().tv_nsec == st.st_mtim.tv_nsec); +#endif } TEST_CASE("Directory") diff --git a/unittest/test_Util.cpp b/unittest/test_Util.cpp index d68f009fe..ac6f30085 100644 --- a/unittest/test_Util.cpp +++ b/unittest/test_Util.cpp @@ -147,6 +147,24 @@ TEST_CASE("Util::ends_with") CHECK_FALSE(Util::ends_with("x", "xy")); } +TEST_CASE("Util::fallocate") +{ + const char* filename = "test-file"; + int fd = creat(filename, S_IRUSR | S_IWUSR); + CHECK(Util::fallocate(fd, 10000) == 0); + close(fd); + fd = open(filename, O_RDWR, S_IRUSR | S_IWUSR); + CHECK(Stat::stat(filename).size() == 10000); + CHECK(Util::fallocate(fd, 5000) == 0); + close(fd); + fd = open(filename, O_RDWR, S_IRUSR | S_IWUSR); + CHECK(Stat::stat(filename).size() == 10000); + CHECK(Util::fallocate(fd, 20000) == 0); + close(fd); + CHECK(Stat::stat(filename).size() == 20000); + unlink(filename); +} + TEST_CASE("Util::for_each_level_1_subdir") { std::vector actual;