From: Joel Rosdahl Date: Wed, 23 Jun 2021 14:29:15 +0000 (+0200) Subject: Add secondary file storage backend X-Git-Tag: v4.4~179 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=f8789e2f4156cd3413f72c0ce4ae0dc321f70084;p=thirdparty%2Fccache.git Add secondary file storage backend Closes #857. --- diff --git a/doc/MANUAL.adoc b/doc/MANUAL.adoc index 7a4cfa535..8e503e86e 100644 --- a/doc/MANUAL.adoc +++ b/doc/MANUAL.adoc @@ -789,6 +789,12 @@ still has to do _some_ preprocessing (like macros). to query after the primary cache storage. See _<<_secondary_storage_backends,Secondary storage backends>>_ for documentation of syntax and available backends. ++ +Examples: ++ +* `file:///shared/nfs/directory` +* `file:///shared/nfs/one|read-only file:///shared/nfs/two` + [[config_sloppiness]] *sloppiness* (*CCACHE_SLOPPINESS*):: By default, ccache tries to give as few false cache hits as possible. @@ -899,6 +905,30 @@ Optional attributes available for all secondary storage backends: * *read-only*: If *true*, only read from this backend, don't write. The default is *false*. +These are the available backends: + +=== File storage backend + +URL format: `file://DIRECTORY` + +This backend stores data as separate files in a directory structure below +*DIRECTORY* (an absolute path), similar (but not identical) to the primary cache +storage. A typical use case for this backend would be sharing a cache on an NFS +directory. Note that ccache will not perform any cleanup of the storage -- that +has to be done by other means. + +Examples: + +* `file:///shared/nfs/directory` +* `file:///shared/nfs/directory|umask=002|update-mtime=true` + +Optional attributes: + +* *umask*: This attribute (an octal integer) overrides the umask to use for + files and directories in the cache directory. +* *update-mtime*: If *true*, update the modification time (mtime) of cache + entries that are read. The default is *false*. + == Cache size management By default, ccache has a 5 GB limit on the total size of files in the cache and @@ -1470,6 +1500,9 @@ systems. One way of improving cache hit rate in that case is to set <> to *system_headers* to ignore system headers. +An alternative to putting the main cache directory on NFS is to set up a +<> file cache. + == Using ccache with other compiler wrappers diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 70694d030..65f7d898e 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -1,4 +1,5 @@ add_subdirectory(primary) +add_subdirectory(secondary) set( sources diff --git a/src/storage/Storage.cpp b/src/storage/Storage.cpp index b39f442b3..32229d06f 100644 --- a/src/storage/Storage.cpp +++ b/src/storage/Storage.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -219,8 +220,13 @@ parse_storage_entry(const nonstd::string_view& entry) } static std::unique_ptr -create_storage(const ParseStorageEntryResult& /*storage_entry*/) +create_storage(const ParseStorageEntryResult& storage_entry) { + if (storage_entry.scheme == "file") { + return std::make_unique(storage_entry.url, + storage_entry.attributes); + } + return {}; } diff --git a/src/storage/secondary/CMakeLists.txt b/src/storage/secondary/CMakeLists.txt new file mode 100644 index 000000000..f6b8c2d66 --- /dev/null +++ b/src/storage/secondary/CMakeLists.txt @@ -0,0 +1,6 @@ +set( + sources + ${CMAKE_CURRENT_SOURCE_DIR}/FileStorage.cpp +) + +target_sources(ccache_lib PRIVATE ${sources}) diff --git a/src/storage/secondary/FileStorage.cpp b/src/storage/secondary/FileStorage.cpp new file mode 100644 index 000000000..677dc3b97 --- /dev/null +++ b/src/storage/secondary/FileStorage.cpp @@ -0,0 +1,157 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "FileStorage.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace storage { +namespace secondary { + +static std::string +parse_url(const std::string& url) +{ + const nonstd::string_view prefix = "file://"; + ASSERT(Util::starts_with(url, prefix)); + const auto dir = url.substr(prefix.size()); + if (!Util::starts_with(dir, "/")) { + throw Error("invalid file URL \"{}\" - directory must start with a slash", + url); + } + return dir; +} + +static nonstd::optional +parse_umask(const AttributeMap& attributes) +{ + const auto it = attributes.find("umask"); + if (it == attributes.end()) { + return nonstd::nullopt; + } + + const auto umask = util::parse_umask(it->second); + if (umask) { + return *umask; + } else { + LOG("Error: {}", umask.error()); + return nonstd::nullopt; + } +} + +static bool +parse_update_mtime(const AttributeMap& attributes) +{ + const auto it = attributes.find("update-mtime"); + return it != attributes.end() && it->second == "true"; +} + +FileStorage::FileStorage(const std::string& url, const AttributeMap& attributes) + : m_dir(parse_url(url)), + m_umask(parse_umask(attributes)), + m_update_mtime(parse_update_mtime(attributes)) +{ +} + +nonstd::expected, SecondaryStorage::Error> +FileStorage::get(const Digest& key) +{ + const auto path = get_entry_path(key); + const bool exists = Stat::stat(path); + + if (!exists) { + // Don't log failure if the entry doesn't exist. + return nonstd::nullopt; + } + + if (m_update_mtime) { + // Update modification timestamp for potential LRU cleanup by some external + // mechanism. + Util::update_mtime(path); + } + + try { + LOG("Reading {}", path); + return Util::read_file(path); + } catch (const ::Error& e) { + LOG("Failed to read {}: {}", path, e.what()); + return nonstd::make_unexpected(Error::error); + } +} + +nonstd::expected +FileStorage::put(const Digest& key, + const std::string& value, + bool only_if_missing) +{ + const auto path = get_entry_path(key); + + if (only_if_missing && Stat::stat(path)) { + LOG("{} already in cache", path); + return false; + } + + { + UmaskScope umask_scope(m_umask); + + util::create_cachedir_tag(m_dir); + + const auto dir = Util::dir_name(path); + if (!Util::create_dir(dir)) { + LOG("Failed to create directory {}: {}", dir, strerror(errno)); + return nonstd::make_unexpected(Error::error); + } + + LOG("Writing {}", path); + try { + AtomicFile file(path, AtomicFile::Mode::binary); + file.write(value); + file.commit(); + return true; + } catch (const ::Error& e) { + LOG("Failed to write {}: {}", path, e.what()); + return nonstd::make_unexpected(Error::error); + } + } +} + +nonstd::expected +FileStorage::remove(const Digest& key) +{ + return Util::unlink_safe(get_entry_path(key)); +} + +std::string +FileStorage::get_entry_path(const Digest& key) const +{ + const auto key_string = key.to_string(); + const uint8_t digits = 2; + return FMT("{}/{:.{}}/{}", m_dir, key_string, digits, &key_string[digits]); +} + +} // namespace secondary +} // namespace storage diff --git a/src/storage/secondary/FileStorage.hpp b/src/storage/secondary/FileStorage.hpp new file mode 100644 index 000000000..18527b3eb --- /dev/null +++ b/src/storage/secondary/FileStorage.hpp @@ -0,0 +1,48 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include + +namespace storage { +namespace secondary { + +class FileStorage : public storage::SecondaryStorage +{ +public: + FileStorage(const std::string& url, const AttributeMap& attributes); + + nonstd::expected, Error> + get(const Digest& key) override; + nonstd::expected put(const Digest& key, + const std::string& value, + bool only_if_missing) override; + nonstd::expected remove(const Digest& key) override; + +private: + const std::string m_dir; + const nonstd::optional m_umask; + const bool m_update_mtime; + + std::string get_entry_path(const Digest& key) const; +}; + +} // namespace secondary +} // namespace storage diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ae0df8e8a..84a9dc161 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -65,6 +65,7 @@ addtest(profiling_hip_clang) addtest(readonly) addtest(readonly_direct) addtest(sanitize_blacklist) +addtest(secondary_file) addtest(serialize_diagnostics) addtest(source_date_epoch) addtest(split_dwarf) diff --git a/test/suites/secondary_file.bash b/test/suites/secondary_file.bash new file mode 100644 index 000000000..953ebba4e --- /dev/null +++ b/test/suites/secondary_file.bash @@ -0,0 +1,125 @@ +SUITE_secondary_file_SETUP() { + unset CCACHE_NODIRECT + export CCACHE_SECONDARY_STORAGE="file://$PWD/secondary" + + generate_code 1 test.c +} + +expect_number_of_cache_entries() { + local expected=$1 + local dir=$2 + local actual + + actual=$(find "$dir" -type f ! -name stats ! -name CACHEDIR.TAG | wc -l) + if [ "$actual" -ne "$expected" ]; then + test_failed_internal "Found $actual (expected $expected) entries in $dir" + fi +} + +SUITE_secondary_file() { + # ------------------------------------------------------------------------- + TEST "Base case" + + $CCACHE_COMPILE -c test.c + expect_stat 'cache hit (direct)' 0 + expect_stat 'cache miss' 1 + expect_stat 'files in cache' 2 + expect_exists secondary/CACHEDIR.TAG + expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest + + $CCACHE_COMPILE -c test.c + expect_stat 'cache hit (direct)' 1 + expect_stat 'cache miss' 1 + expect_stat 'files in cache' 2 + expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest + + $CCACHE -C >/dev/null + expect_stat 'files in cache' 0 + expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest + + $CCACHE_COMPILE -c test.c + expect_stat 'cache hit (direct)' 2 + expect_stat 'cache miss' 1 + expect_stat 'files in cache' 0 + expect_stat 'files in cache' 0 + expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest + + # ------------------------------------------------------------------------- + TEST "Two directories" + + CCACHE_SECONDARY_STORAGE+=" file://$PWD/secondary_2" + mkdir secondary_2 + + $CCACHE_COMPILE -c test.c + expect_stat 'cache hit (direct)' 0 + expect_stat 'cache miss' 1 + expect_stat 'files in cache' 2 + expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest + expect_file_count 3 '*' secondary_2 # CACHEDIR.TAG + result + manifest + + $CCACHE -C >/dev/null + expect_stat 'files in cache' 0 + expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest + expect_file_count 3 '*' secondary_2 # CACHEDIR.TAG + result + manifest + + $CCACHE_COMPILE -c test.c + expect_stat 'cache hit (direct)' 1 + expect_stat 'cache miss' 1 + expect_stat 'files in cache' 0 + expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest + expect_file_count 3 '*' secondary_2 # CACHEDIR.TAG + result + manifest + + rm -r secondary/?? + expect_file_count 1 '*' secondary # CACHEDIR.TAG + + $CCACHE_COMPILE -c test.c + expect_stat 'cache hit (direct)' 2 + expect_stat 'cache miss' 1 + expect_stat 'files in cache' 0 + expect_file_count 1 '*' secondary # CACHEDIR.TAG + expect_file_count 3 '*' secondary_2 # CACHEDIR.TAG + result + manifest + + # ------------------------------------------------------------------------- + TEST "Read-only" + + $CCACHE_COMPILE -c test.c + expect_stat 'cache hit (direct)' 0 + expect_stat 'cache miss' 1 + expect_stat 'files in cache' 2 + expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest + + $CCACHE -C >/dev/null + expect_stat 'files in cache' 0 + expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest + + CCACHE_SECONDARY_STORAGE+="|read-only" + + $CCACHE_COMPILE -c test.c + expect_stat 'cache hit (direct)' 1 + expect_stat 'cache miss' 1 + expect_stat 'files in cache' 0 + expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest + + echo 'int x;' >> test.c + $CCACHE_COMPILE -c test.c + expect_stat 'cache hit (direct)' 1 + expect_stat 'cache miss' 2 + expect_stat 'files in cache' 2 + expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest + + # ------------------------------------------------------------------------- + TEST "umask" + + CCACHE_SECONDARY_STORAGE="file://$PWD/secondary|umask=022" + rm -rf secondary + $CCACHE_COMPILE -c test.c + expect_perm secondary drwxr-xr-x + expect_perm secondary/CACHEDIR.TAG -rw-r--r-- + + CCACHE_SECONDARY_STORAGE="file://$PWD/secondary|umask=000" + $CCACHE -C >/dev/null + rm -rf secondary + $CCACHE_COMPILE -c test.c + expect_perm secondary drwxrwxrwx + expect_perm secondary/CACHEDIR.TAG -rw-rw-rw- +}