Closes #857.
to query after the primary cache storage. See
_<<_secondary_storage_backends,Secondary storage backends>>_ for
documentation of syntax and available backends.
++
+Examples:
++
+* `file:///shared/nfs/directory`
+* `file:///shared/nfs/one|read-only file:///shared/nfs/two`
+
[[config_sloppiness]] *sloppiness* (*CCACHE_SLOPPINESS*)::
By default, ccache tries to give as few false cache hits as possible.
* *read-only*: If *true*, only read from this backend, don't write. The default
is *false*.
+These are the available backends:
+
+=== File storage backend
+
+URL format: `file://DIRECTORY`
+
+This backend stores data as separate files in a directory structure below
+*DIRECTORY* (an absolute path), similar (but not identical) to the primary cache
+storage. A typical use case for this backend would be sharing a cache on an NFS
+directory. Note that ccache will not perform any cleanup of the storage -- that
+has to be done by other means.
+
+Examples:
+
+* `file:///shared/nfs/directory`
+* `file:///shared/nfs/directory|umask=002|update-mtime=true`
+
+Optional attributes:
+
+* *umask*: This attribute (an octal integer) overrides the umask to use for
+ files and directories in the cache directory.
+* *update-mtime*: If *true*, update the modification time (mtime) of cache
+ entries that are read. The default is *false*.
+
== Cache size management
By default, ccache has a 5 GB limit on the total size of files in the cache and
<<config_sloppiness,*sloppiness*>> to *system_headers* to ignore system
headers.
+An alternative to putting the main cache directory on NFS is to set up a
+<<config_secondary_storage,secondary storage>> file cache.
+
== Using ccache with other compiler wrappers
add_subdirectory(primary)
+add_subdirectory(secondary)
set(
sources
#include <Util.hpp>
#include <assertions.hpp>
#include <fmtmacros.hpp>
+#include <storage/secondary/FileStorage.hpp>
#include <util/Tokenizer.hpp>
#include <util/string_utils.hpp>
}
static std::unique_ptr<SecondaryStorage>
-create_storage(const ParseStorageEntryResult& /*storage_entry*/)
+create_storage(const ParseStorageEntryResult& storage_entry)
{
+ if (storage_entry.scheme == "file") {
+ return std::make_unique<secondary::FileStorage>(storage_entry.url,
+ storage_entry.attributes);
+ }
+
return {};
}
--- /dev/null
+set(
+ sources
+ ${CMAKE_CURRENT_SOURCE_DIR}/FileStorage.cpp
+)
+
+target_sources(ccache_lib PRIVATE ${sources})
--- /dev/null
+// Copyright (C) 2021 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "FileStorage.hpp"
+
+#include <AtomicFile.hpp>
+#include <Digest.hpp>
+#include <Logging.hpp>
+#include <UmaskScope.hpp>
+#include <Util.hpp>
+#include <assertions.hpp>
+#include <fmtmacros.hpp>
+#include <util/file_utils.hpp>
+#include <util/string_utils.hpp>
+
+#include <third_party/nonstd/string_view.hpp>
+
+namespace storage {
+namespace secondary {
+
+static std::string
+parse_url(const std::string& url)
+{
+ const nonstd::string_view prefix = "file://";
+ ASSERT(Util::starts_with(url, prefix));
+ const auto dir = url.substr(prefix.size());
+ if (!Util::starts_with(dir, "/")) {
+ throw Error("invalid file URL \"{}\" - directory must start with a slash",
+ url);
+ }
+ return dir;
+}
+
+static nonstd::optional<mode_t>
+parse_umask(const AttributeMap& attributes)
+{
+ const auto it = attributes.find("umask");
+ if (it == attributes.end()) {
+ return nonstd::nullopt;
+ }
+
+ const auto umask = util::parse_umask(it->second);
+ if (umask) {
+ return *umask;
+ } else {
+ LOG("Error: {}", umask.error());
+ return nonstd::nullopt;
+ }
+}
+
+static bool
+parse_update_mtime(const AttributeMap& attributes)
+{
+ const auto it = attributes.find("update-mtime");
+ return it != attributes.end() && it->second == "true";
+}
+
+FileStorage::FileStorage(const std::string& url, const AttributeMap& attributes)
+ : m_dir(parse_url(url)),
+ m_umask(parse_umask(attributes)),
+ m_update_mtime(parse_update_mtime(attributes))
+{
+}
+
+nonstd::expected<nonstd::optional<std::string>, SecondaryStorage::Error>
+FileStorage::get(const Digest& key)
+{
+ const auto path = get_entry_path(key);
+ const bool exists = Stat::stat(path);
+
+ if (!exists) {
+ // Don't log failure if the entry doesn't exist.
+ return nonstd::nullopt;
+ }
+
+ if (m_update_mtime) {
+ // Update modification timestamp for potential LRU cleanup by some external
+ // mechanism.
+ Util::update_mtime(path);
+ }
+
+ try {
+ LOG("Reading {}", path);
+ return Util::read_file(path);
+ } catch (const ::Error& e) {
+ LOG("Failed to read {}: {}", path, e.what());
+ return nonstd::make_unexpected(Error::error);
+ }
+}
+
+nonstd::expected<bool, SecondaryStorage::Error>
+FileStorage::put(const Digest& key,
+ const std::string& value,
+ bool only_if_missing)
+{
+ const auto path = get_entry_path(key);
+
+ if (only_if_missing && Stat::stat(path)) {
+ LOG("{} already in cache", path);
+ return false;
+ }
+
+ {
+ UmaskScope umask_scope(m_umask);
+
+ util::create_cachedir_tag(m_dir);
+
+ const auto dir = Util::dir_name(path);
+ if (!Util::create_dir(dir)) {
+ LOG("Failed to create directory {}: {}", dir, strerror(errno));
+ return nonstd::make_unexpected(Error::error);
+ }
+
+ LOG("Writing {}", path);
+ try {
+ AtomicFile file(path, AtomicFile::Mode::binary);
+ file.write(value);
+ file.commit();
+ return true;
+ } catch (const ::Error& e) {
+ LOG("Failed to write {}: {}", path, e.what());
+ return nonstd::make_unexpected(Error::error);
+ }
+ }
+}
+
+nonstd::expected<bool, SecondaryStorage::Error>
+FileStorage::remove(const Digest& key)
+{
+ return Util::unlink_safe(get_entry_path(key));
+}
+
+std::string
+FileStorage::get_entry_path(const Digest& key) const
+{
+ const auto key_string = key.to_string();
+ const uint8_t digits = 2;
+ return FMT("{}/{:.{}}/{}", m_dir, key_string, digits, &key_string[digits]);
+}
+
+} // namespace secondary
+} // namespace storage
--- /dev/null
+// Copyright (C) 2021 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#pragma once
+
+#include <storage/SecondaryStorage.hpp>
+#include <storage/types.hpp>
+
+namespace storage {
+namespace secondary {
+
+class FileStorage : public storage::SecondaryStorage
+{
+public:
+ FileStorage(const std::string& url, const AttributeMap& attributes);
+
+ nonstd::expected<nonstd::optional<std::string>, Error>
+ get(const Digest& key) override;
+ nonstd::expected<bool, Error> put(const Digest& key,
+ const std::string& value,
+ bool only_if_missing) override;
+ nonstd::expected<bool, Error> remove(const Digest& key) override;
+
+private:
+ const std::string m_dir;
+ const nonstd::optional<mode_t> m_umask;
+ const bool m_update_mtime;
+
+ std::string get_entry_path(const Digest& key) const;
+};
+
+} // namespace secondary
+} // namespace storage
addtest(readonly)
addtest(readonly_direct)
addtest(sanitize_blacklist)
+addtest(secondary_file)
addtest(serialize_diagnostics)
addtest(source_date_epoch)
addtest(split_dwarf)
--- /dev/null
+SUITE_secondary_file_SETUP() {
+ unset CCACHE_NODIRECT
+ export CCACHE_SECONDARY_STORAGE="file://$PWD/secondary"
+
+ generate_code 1 test.c
+}
+
+expect_number_of_cache_entries() {
+ local expected=$1
+ local dir=$2
+ local actual
+
+ actual=$(find "$dir" -type f ! -name stats ! -name CACHEDIR.TAG | wc -l)
+ if [ "$actual" -ne "$expected" ]; then
+ test_failed_internal "Found $actual (expected $expected) entries in $dir"
+ fi
+}
+
+SUITE_secondary_file() {
+ # -------------------------------------------------------------------------
+ TEST "Base case"
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat 'cache hit (direct)' 0
+ expect_stat 'cache miss' 1
+ expect_stat 'files in cache' 2
+ expect_exists secondary/CACHEDIR.TAG
+ expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat 'cache hit (direct)' 1
+ expect_stat 'cache miss' 1
+ expect_stat 'files in cache' 2
+ expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
+
+ $CCACHE -C >/dev/null
+ expect_stat 'files in cache' 0
+ expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat 'cache hit (direct)' 2
+ expect_stat 'cache miss' 1
+ expect_stat 'files in cache' 0
+ expect_stat 'files in cache' 0
+ expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
+
+ # -------------------------------------------------------------------------
+ TEST "Two directories"
+
+ CCACHE_SECONDARY_STORAGE+=" file://$PWD/secondary_2"
+ mkdir secondary_2
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat 'cache hit (direct)' 0
+ expect_stat 'cache miss' 1
+ expect_stat 'files in cache' 2
+ expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
+ expect_file_count 3 '*' secondary_2 # CACHEDIR.TAG + result + manifest
+
+ $CCACHE -C >/dev/null
+ expect_stat 'files in cache' 0
+ expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
+ expect_file_count 3 '*' secondary_2 # CACHEDIR.TAG + result + manifest
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat 'cache hit (direct)' 1
+ expect_stat 'cache miss' 1
+ expect_stat 'files in cache' 0
+ expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
+ expect_file_count 3 '*' secondary_2 # CACHEDIR.TAG + result + manifest
+
+ rm -r secondary/??
+ expect_file_count 1 '*' secondary # CACHEDIR.TAG
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat 'cache hit (direct)' 2
+ expect_stat 'cache miss' 1
+ expect_stat 'files in cache' 0
+ expect_file_count 1 '*' secondary # CACHEDIR.TAG
+ expect_file_count 3 '*' secondary_2 # CACHEDIR.TAG + result + manifest
+
+ # -------------------------------------------------------------------------
+ TEST "Read-only"
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat 'cache hit (direct)' 0
+ expect_stat 'cache miss' 1
+ expect_stat 'files in cache' 2
+ expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
+
+ $CCACHE -C >/dev/null
+ expect_stat 'files in cache' 0
+ expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
+
+ CCACHE_SECONDARY_STORAGE+="|read-only"
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat 'cache hit (direct)' 1
+ expect_stat 'cache miss' 1
+ expect_stat 'files in cache' 0
+ expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
+
+ echo 'int x;' >> test.c
+ $CCACHE_COMPILE -c test.c
+ expect_stat 'cache hit (direct)' 1
+ expect_stat 'cache miss' 2
+ expect_stat 'files in cache' 2
+ expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
+
+ # -------------------------------------------------------------------------
+ TEST "umask"
+
+ CCACHE_SECONDARY_STORAGE="file://$PWD/secondary|umask=022"
+ rm -rf secondary
+ $CCACHE_COMPILE -c test.c
+ expect_perm secondary drwxr-xr-x
+ expect_perm secondary/CACHEDIR.TAG -rw-r--r--
+
+ CCACHE_SECONDARY_STORAGE="file://$PWD/secondary|umask=000"
+ $CCACHE -C >/dev/null
+ rm -rf secondary
+ $CCACHE_COMPILE -c test.c
+ expect_perm secondary drwxrwxrwx
+ expect_perm secondary/CACHEDIR.TAG -rw-rw-rw-
+}
projects / thirdparty / ccache.git / commitdiff
