From 7f40418930cae9f86f5e80fd92b6d79baf621d34 Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Sun, 18 Jan 2026 14:48:06 +0100 Subject: [PATCH] feat: Add support for remote storage helpers This commit adds support for communicating with a remote storage server using a long-lived local helper process, started by ccache on demand. The helper process can keep connections alive (thus amortizing the session setup cost), and knowledge about remote storage protocols can be kept out of the ccache code base, making it possible to develop and release support for different storage server protocols independently. The storage helper listens to a Unix-domain socket on POSIX systems and a named pipe on Windows systems. Ccache communicates with it using a custom IPC protocol, described in doc/remote_storage_helper_spec.md. The helper is named ccache-storage- (e.g. ccache-storage-https) and can be placed in $PATH, next to the ccache executable or in ccache's libexec directory. Storage helpers time terminate after a while on inactivity. Built-in support for http and redis is kept for now but will likely be removed in a future ccache release. - The syntax of the remote_storage/CCACHE_REMOTE_STORAGE configuration option has been improved (in a backward compatible way): What used to be called "attributes" are now split into "properties" and "custom attributes", where properties are things that ccache knows about and acts on (e.g. read-only and shards) while attributes are custom key-value pairs sent to the helper without ccache knowing or caring about them. Syntax: A property is "key=value", a custom attribute is "@key=value". - Properties/attributes are now separated by whitespace instead of pipe characters. New remote storage properties: - data-timeout: Timeout for send/receive data transfer. Resets whenever data is received or can be sent. Default: 1s. - request-timeout: Timeout for the whole request. Default: 10s. - idle-timeout: Timeout for the helper process to wait before exiting after client inactivity (0s: stay up indefinitely). Default: 10m. - helper: Override which storage helper to spawn (either a filename or a full path). This is mainly useful when developing or testing new helpers. The new command line option "--stop-storage-helpers" can be used to ask spawned storage helpers to stop immediately. A new "libexec_dirs" configuration option is available for overriding the default libexec determined at build time. --- ci/prepare-release | 1 + cmake/GenerateConfigurationFile.cmake | 3 +- cmake/config.h.in | 1 + doc/manual.adoc | 136 ++-- misc/Makefile.posix-binary-release | 4 +- misc/patch-binary.py | 18 + src/ccache/ccache.cpp | 16 +- src/ccache/config.cpp | 24 + src/ccache/config.hpp | 11 + src/ccache/context.cpp | 4 +- src/ccache/context.hpp | 2 +- src/ccache/core/mainoptions.cpp | 87 +-- src/ccache/storage/remote/CMakeLists.txt | 1 + src/ccache/storage/remote/filestorage.cpp | 2 +- src/ccache/storage/remote/helper.cpp | 654 ++++++++++++++++++++ src/ccache/storage/remote/helper.hpp | 55 ++ src/ccache/storage/remote/httpstorage.cpp | 15 +- src/ccache/storage/remote/httpstorage.hpp | 5 +- src/ccache/storage/remote/redisstorage.cpp | 2 +- src/ccache/storage/remote/remotestorage.cpp | 8 +- src/ccache/storage/remote/remotestorage.hpp | 24 +- src/ccache/storage/storage.cpp | 392 ++++++++---- src/ccache/storage/storage.hpp | 12 +- test/CMakeLists.txt | 1 + test/run | 4 +- test/suites/remote_file.bash | 16 +- test/suites/remote_helper.bash | 157 +++++ test/suites/remote_http.bash | 16 +- test/suites/remote_url.bash | 14 - unittest/test_config.cpp | 2 + 30 files changed, 1421 insertions(+), 266 deletions(-) create mode 100755 misc/patch-binary.py create mode 100644 src/ccache/storage/remote/helper.cpp create mode 100644 src/ccache/storage/remote/helper.hpp create mode 100644 test/suites/remote_helper.bash diff --git a/ci/prepare-release b/ci/prepare-release index e14e3b006..dfb7aa85b 100755 --- a/ci/prepare-release +++ b/ci/prepare-release @@ -31,6 +31,7 @@ prepare_posix_binary_release() { cp "${arch}-binary/ccache" "${name}" chmod +x "${name}/ccache" cp misc/Makefile.posix-binary-release "${name}/Makefile" + cp misc/patch-binary.py "${name}" cp GPL-3.0.txt README.md "${name}" cp docs/install/usr/local/share/doc/ccache/* "${name}" cp docs/install/usr/local/share/man/man1/ccache.1 "${name}" diff --git a/cmake/GenerateConfigurationFile.cmake b/cmake/GenerateConfigurationFile.cmake index fcc7f0252..01522ad9c 100644 --- a/cmake/GenerateConfigurationFile.cmake +++ b/cmake/GenerateConfigurationFile.cmake @@ -100,7 +100,8 @@ if(WIN32) set(INODE_CACHE_SUPPORTED 1) endif() -# Escape backslashes in SYSCONFDIR for C. +file(TO_NATIVE_PATH "${CMAKE_INSTALL_FULL_LIBEXECDIR}" CONFIG_LIBEXECDIR_C_ESCAPED) +string(REPLACE "\\" "\\\\" CONFIG_LIBEXECDIR_C_ESCAPED "${CONFIG_LIBEXECDIR_C_ESCAPED}") file(TO_NATIVE_PATH "${CMAKE_INSTALL_FULL_SYSCONFDIR}" CONFIG_SYSCONFDIR_C_ESCAPED) string(REPLACE "\\" "\\\\" CONFIG_SYSCONFDIR_C_ESCAPED "${CONFIG_SYSCONFDIR_C_ESCAPED}") diff --git a/cmake/config.h.in b/cmake/config.h.in index f7228d539..dfd8e4e33 100644 --- a/cmake/config.h.in +++ b/cmake/config.h.in @@ -209,6 +209,7 @@ typedef int pid_t; # define ESTALE -1 #endif +#define LIBEXECDIR "@CONFIG_LIBEXECDIR_C_ESCAPED@" #define SYSCONFDIR "@CONFIG_SYSCONFDIR_C_ESCAPED@" #cmakedefine INODE_CACHE_SUPPORTED diff --git a/doc/manual.adoc b/doc/manual.adoc index 3cbcb0b6f..30bf815b4 100644 --- a/doc/manual.adoc +++ b/doc/manual.adoc @@ -178,6 +178,11 @@ should use the normal compiler options and refer to your compiler's documentatio format. Use `-v`/`--verbose` once or twice for more details. See <<_cache_statistics,Cache statistics>> for more information. +*--stop-storage-helpers*:: + + Storage helpers normally stop after a period of client inactivity, but this + option can be used to stop any running helpers immediately. + *--threads* _THREADS_:: Use up to _THREADS_ threads for threaded operations. The default is to use @@ -864,6 +869,13 @@ the default is false. output. The default is false. This can be used to check documentation with `-Wdocumentation`. +[#config_libexec_dir] +*libexec_dirs* (*CCACHE_LIBEXEC_DIRS*):: + + If set, ccache will look for helper programs in these directories instead of + the default. The list separator is semicolon on Windows systems and colon on + other systems. + [#config_log_file] *log_file* (*CCACHE_LOGFILE*):: @@ -1007,7 +1019,7 @@ Examples: ---- remote_storage = file:/shared/nfs/directory+` remote_storage = - file:///shared/nfs/one|read-only + file:///shared/nfs/one read-only file:///shared/nfs/two remote_storage = file:///Z:/example/windows/folder remote_storage = http://example.com/cache @@ -1207,18 +1219,46 @@ true to disable local storage. Note that cache statistics counters will still be kept in the local cache directory -- remote storage backends only store compilation results and manifests. -A remote storage backend is specified with a URL, optionally followed by a pipe -(`|`) and a pipe-separated list of attributes. An attribute is _key_=_value_ or -just _key_ as a short form of _key_=*true*. Attribute values must be -https://en.wikipedia.org/wiki/Percent-encoding[percent-encoded] if they contain -percent, pipe or space characters. +=== Storage helper process + +ccache can spawn a long-lived local storage helper process (installed +separately) to handle communication with the remote storage. Storage helpers +keep remote connections up for efficiency and terminate after a while on +inactivity. + +The storage helper program is called `ccache-storage-`, where `` +is the scheme part of a URL, e.g. `https`. The program can be placed in these +locations (in priority order): -=== Common attributes +1. In a location specified by the *helper* property (see below). +2. In `$PATH`. +3. Next to the ccache executable. +4. In ccache's <>. -These optional attributes are available for all remote storage backends: +If the program is not found, ccache falls back to one of the builtin backends if +available, otherwise ccache exits with an error. +=== Configuration syntax + +A remote storage backend is specified by a URL and can be followed by a +whitespace-separated list of properties (`_key_=_value_`) and custom attributes +(`@_key_=_value_`). A missing `=_value_` is shorthand for setting the value to +*true*. Values must be +https://en.wikipedia.org/wiki/Percent-encoding[percent-encoded] if they contain +`%`, `|` or whitespace characters. For compatibility with older ccache versions, +`|` characters are treated as spaces. + +==== Properties + +* *data-timeout*: Timeout for send/receive data transfer. Resets whenever data + is received or can be sent. Default: 1s. +* *helper*: Override which storage helper to spawn (either a filename or a full + path). This is mainly useful when developing or testing new helpers. +* *idle-timeout*: Timeout for the helper process to wait before exiting after + client inactivity (0s: stay up indefinitely). Default: 10m. * *read-only*: If *true*, only read from this backend, don't write. The default is *false*. +* *request-timeout*: Timeout for the whole request. Default: 10s. * *shards*: A comma-separated list of names for sharding (partitioning) the cache entries using https://en.wikipedia.org/wiki/Rendezvous_hashing[Rendezvous hashing], @@ -1231,17 +1271,28 @@ These optional attributes are available for all remote storage backends: to represent the available memory for a memory cache on a specific server. The default weight is *1*. + +Timeouts can be specified with an ms (milliseconds), s (seconds), m (minutes), h +(hours) or d (days) suffix. ++ Examples: + -- -* `+redis://cache-*.example.com|shards=a(3),b(1),c(1.5)+` will put 55% (3/5.5) - of the cache on `+redis://cache-a.example.com+`, 18% (1/5.5) on - `+redis://cache-b.example.com+` and 27% (1.5/5.5) on - `+redis://cache-c.example.com+`. -* `+http://example.com/*|shards=alpha,beta+` will put 50% of the cache on +* `+http://cache-*.example.com shards=a(3),b(1),c(1.5)+` will put 55% (3/5.5) + of the cache on `+http://cache-a.example.com+`, 18% (1/5.5) on + `+http://cache-b.example.com+` and 27% (1.5/5.5) on + `+http://cache-c.example.com+`. +* `+http://example.com/* shards=alpha,beta+` will put 50% of the cache on `+http://example.com/alpha+` and 50% on `+http://example.com/beta+`. -- +==== Custom attributes + +Custom attributes on the form `_@key_=_value_` are specific to each storage +helper implementation. See the backend's documentation for which attributes are +available. Examples: + +* `+file:///example/directory @umask=002 @update-mtime=true+` +* `+https://example.com/path @bearer-token=deadbeef+` === Storage interaction @@ -1259,7 +1310,7 @@ the default): |============================================================================== -^[1]^ Unless remote storage has attribute `read-only=true`. + +^[1]^ Unless remote storage has property `read-only=true`. + ^[2]^ Unless local storage is set to share its cache hits with the <> option. @@ -1291,13 +1342,13 @@ done by other means, for instance by running `ccache --trim-dir` periodically. Examples: * `+file:/shared/nfs/directory+` -* `+file:///shared/nfs/directory|umask=002|update-mtime=true+` +* `+file:///shared/nfs/directory @umask=002 @update-mtime=true+` * `+file:///Z:/example/windows/folder+` * `+file://example.com/shared/ccache%20folder+` Optional attributes: -* *layout*: How to store file under the cache directory. Available values: +* *@layout*: How to store file under the cache directory. Available values: + -- * *flat*: Store all files directly under the cache directory. @@ -1305,14 +1356,33 @@ Optional attributes: -- + The default is *subdirs*. -* *umask*: This attribute (an octal integer) overrides the umask to use for +* *@umask*: This attribute (an octal integer) overrides the umask to use for files and directories in the cache directory. -* *update-mtime*: If *true*, update the modification time (mtime) of cache +* *@update-mtime*: If *true*, update the modification time (mtime) of cache entries that are read. The default is *false*. +=== CRSH storage backend + +URL format: `+crsh:IPC_ENDPOINT+` + +This backend connects to an already running <> (CRSH) process listening on `IPC_ENDPOINT` (Unix socket path or +Windows named pipe). The use case for this is mainly when developing or +debugging a storage helper, or in special setups where the helper program is +started and managed externally. + +Examples: + +* `+crsh:/path/to/your.sock+` (Unix socket) +* `+crsh:name-of-your-pipe+` (Windows named pipe) + + === HTTP storage backend +WARNING: Built-in support for the HTTP protocol will likely be removed in a +future ccache release. + URL format: `+http://HOST[:PORT][/PATH]+` This backend stores data in an HTTP-compatible server. The required HTTP methods @@ -1330,15 +1400,14 @@ Examples: * `+http://localhost+` * `+http://someusername:p4ssw0rd@example.com/cache/+` -* `+http://localhost:8080|layout=bazel|connect-timeout=50+` +* `+http://localhost:8080 @layout=bazel+` Optional attributes: -* *bearer-token*: Bearer token used to authorize the HTTP requests. -* *connect-timeout*: Timeout (in ms) for network connection. The default is 100. -* *keep-alive*: If *true*, keep the HTTP connection to the storage server open +* *@bearer-token*: Bearer token used to authorize the HTTP requests. +* *@keep-alive*: If *true*, keep the HTTP connection to the storage server open to avoid reconnects. The default is *true*. -* *layout*: How to map key names to the path part of the URL. Available values: +* *@layout*: How to map key names to the path part of the URL. Available values: + -- * *bazel*: Store values in a format compatible with the Bazel HTTP caching @@ -1356,14 +1425,15 @@ values. -- + The default is *subdirs*. -* *header*: Add the key=value pair to the HTTP headers of the request. For example: - `+header=Content-Type=application/octet-stream+` adds - "Content-Type: application/octet-stream" to the http headers of the request. -* *operation-timeout*: Timeout (in ms) for HTTP requests. The default is 10000. - +* *@header*: Add the key=value pair to the HTTP headers of the request. For + example: `+@header=Content-Type=application/octet-stream+` adds + "Content-Type: application/octet-stream" to the HTTP headers of the request. === Redis storage backend +WARNING: Built-in support for the Redis protocol will likely be removed in a +future ccache release. + URL formats: `+redis://[[USERNAME:]PASSWORD@]HOST[:PORT][/DBNUMBER]+` + @@ -1380,23 +1450,17 @@ https://redis.io/topics/lru-cache[configure LRU eviction]. TIP: See https://ccache.dev/howto/redis-storage.html[How to set up Redis storage] for hints on setting up a Redis server for use with ccache. -TIP: You can set up a cluster of Redis servers using the `shards` attribute +TIP: You can set up a cluster of Redis servers using the `shards` property described in _<>_. Examples: * `+redis://localhost+` -* `+redis://p4ssw0rd@cache.example.com:6379/0|connect-timeout=50+` +* `+redis://p4ssw0rd@cache.example.com:6379/0+` * `+redis+unix:/run/redis.sock+` * `+redis+unix:///run/redis.sock+` * `+redis+unix://p4ssw0rd@localhost/run/redis.sock?db=0+` -Optional attributes: - -* *connect-timeout*: Timeout (in ms) for network connection. The default is 100. -* *operation-timeout*: Timeout (in ms) for Redis commands. The default is 10000. - - == Cache size management By default, ccache has a 5 GB limit on the total size of files in the cache and diff --git a/misc/Makefile.posix-binary-release b/misc/Makefile.posix-binary-release index 546a809d2..e7bd8c097 100644 --- a/misc/Makefile.posix-binary-release +++ b/misc/Makefile.posix-binary-release @@ -3,11 +3,11 @@ exec_prefix = $(prefix) bindir = $(exec_prefix)/bin datarootdir = $(prefix)/share docdir = $(datarootdir)/doc/ccache +libexecdir = $(exec_prefix)/libexec mandir = $(datarootdir)/man man1dir = $(mandir)/man1 sysconfdir = $(prefix)/etc -default_sysconfdir = /usr/local/etc doc_files = \ GPL-3.0.txt \ LICENSE.html \ @@ -29,7 +29,7 @@ all: install: mkdir -p "$(DESTDIR)$(bindir)" - $(PYTHON) -c 'import sys; sysconfdir = b"$(sysconfdir)"; default_sysconfdir = b"$(default_sysconfdir)"; sys.stdout.buffer.write(sys.stdin.buffer.read().replace(default_sysconfdir + b"\x00" * (4096 - len(default_sysconfdir)), sysconfdir + b"\x00" * (4096 - len(sysconfdir))))' "$(DESTDIR)$(bindir)/ccache" + $(PYTHON) patch-binary.py "$(libexecdir)" "$(sysconfdir)" "$(DESTDIR)$(bindir)/ccache" chmod +x "$(DESTDIR)$(bindir)/ccache" mkdir -p "$(DESTDIR)$(docdir)" diff --git a/misc/patch-binary.py b/misc/patch-binary.py new file mode 100755 index 000000000..7c8fdb0e3 --- /dev/null +++ b/misc/patch-binary.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +import sys + +libexecdir = sys.argv[1] +sysconfdir = sys.argv[2] + +replacements = [ + (b"/usr/local/libexec", libexecdir.encode()), + (b"/usr/local/etc", sysconfdir.encode()), +] +ccache_bin = sys.stdin.buffer.read() +for repl_from, repl_to in replacements: + ccache_bin = ccache_bin.replace( + repl_from + b"\x00" * (4096 - len(repl_from)), + repl_to + b"\x00" * (4096 - len(repl_to)), + ) +sys.stdout.buffer.write(ccache_bin) diff --git a/src/ccache/ccache.cpp b/src/ccache/ccache.cpp index efcd95b5a..51db104c2 100644 --- a/src/ccache/ccache.cpp +++ b/src/ccache/ccache.cpp @@ -2671,8 +2671,6 @@ initialize(Context& ctx, const char* const* argv, bool masquerading_as_compiler) } } - ctx.storage.initialize(); - find_compiler(ctx, &find_non_ccache_executable, masquerading_as_compiler); // Guess compiler after logging the config value in order to be able to @@ -2786,7 +2784,15 @@ cache_compilation(int argc, const char* const* argv) } { - Context ctx; + fs::path argv0 = +#ifdef _WIN32 + util::add_exe_suffix(argv[0]); +#else + argv[0]; +#endif + const auto ccache_exe_dir = + fs::canonical(argv0).value_or(argv0).parent_path(); + Context ctx(ccache_exe_dir); ctx.initialize(std::move(argv_parts.compiler_and_args), argv_parts.config_settings); SignalHandler signal_handler(ctx); @@ -2803,7 +2809,7 @@ cache_compilation(int argc, const char* const* argv) if (!ctx.config.remote_only()) { ctx.storage.local.increment_statistic(Statistic::local_storage_miss); } - if (ctx.storage.has_remote_storage()) { + if (!ctx.config.remote_storage().empty()) { ctx.storage.local.increment_statistic(Statistic::remote_storage_miss); } } else if ((counters.get(Statistic::direct_cache_hit) > 0 @@ -2903,7 +2909,7 @@ do_cache_compilation(Context& ctx) ctx.config.set_depend_mode(false); } - if (ctx.storage.has_remote_storage()) { + if (!ctx.config.remote_storage().empty()) { if (ctx.config.file_clone()) { LOG_RAW("Disabling file clone mode since remote storage is enabled"); ctx.config.set_file_clone(false); diff --git a/src/ccache/config.cpp b/src/ccache/config.cpp index c81d09514..7ccdf9f3d 100644 --- a/src/ccache/config.cpp +++ b/src/ccache/config.cpp @@ -63,6 +63,8 @@ DLLIMPORT extern char** environ; // Make room for binary patching at install time. The extra pointer to a buffer // is needed to prevent the compiler from assuming too much about the string, // such as its actual length. +const char k_libexecdir_array[4096 + 1] = LIBEXECDIR; +const char* k_libexecdir = k_libexecdir_array; const char k_sysconfdir_array[4096 + 1] = SYSCONFDIR; const char* k_sysconfdir = k_sysconfdir_array; @@ -97,6 +99,7 @@ enum class ConfigItem : uint8_t { ignore_options, inode_cache, keep_comments_cpp, + libexec_dirs, log_file, max_files, max_size, @@ -154,6 +157,7 @@ const std::unordered_map {"ignore_options", {ConfigItem::ignore_options} }, {"inode_cache", {ConfigItem::inode_cache} }, {"keep_comments_cpp", {ConfigItem::keep_comments_cpp} }, + {"libexec_dirs", {ConfigItem::libexec_dirs} }, {"log_file", {ConfigItem::log_file} }, {"max_files", {ConfigItem::max_files} }, {"max_size", {ConfigItem::max_size} }, @@ -205,6 +209,7 @@ const std::unordered_map {"IGNOREHEADERS", "ignore_headers_in_manifest"}, {"IGNOREOPTIONS", "ignore_options" }, {"INODECACHE", "inode_cache" }, + {"LIBEXEC_DIRS", "libexec_dirs" }, {"LOGFILE", "log_file" }, {"MAXFILES", "max_files" }, {"MAXSIZE", "max_size" }, @@ -669,6 +674,18 @@ Config::read(const std::vector& cmdline_config_settings) // system config). } +const char* +Config::libexec_dir() +{ + return k_libexecdir; +} + +const char* +Config::sysconf_dir() +{ + return k_sysconfdir; +} + const fs::path& Config::config_path() const { @@ -855,6 +872,9 @@ Config::get_string_value(const std::string& key) const case ConfigItem::keep_comments_cpp: return format_bool(m_keep_comments_cpp); + case ConfigItem::libexec_dirs: + return util::join_path_list(m_libexec_dirs); + case ConfigItem::log_file: return m_log_file.string(); @@ -1134,6 +1154,10 @@ Config::set_item(const std::string_view& key, m_keep_comments_cpp = parse_bool(value, env_var_key, negate); break; + case ConfigItem::libexec_dirs: + m_libexec_dirs = util::split_path_list(value); + break; + case ConfigItem::log_file: m_log_file = value; break; diff --git a/src/ccache/config.hpp b/src/ccache/config.hpp index e2f10178b..740c885df 100644 --- a/src/ccache/config.hpp +++ b/src/ccache/config.hpp @@ -57,6 +57,9 @@ public: void read(const std::vector& cmdline_config_settings = {}); + static const char* libexec_dir(); + static const char* sysconf_dir(); + bool absolute_paths_in_stderr() const; util::Args::ResponseFileFormat response_file_format() const; const std::vector& base_dirs() const; @@ -81,6 +84,7 @@ public: const std::string& ignore_options() const; bool inode_cache() const; bool keep_comments_cpp() const; + const std::vector& libexec_dirs() const; const std::filesystem::path& log_file() const; uint64_t max_files() const; uint64_t max_size() const; @@ -206,6 +210,7 @@ private: #endif bool m_keep_comments_cpp = false; std::filesystem::path m_log_file; + std::vector m_libexec_dirs{libexec_dir()}; uint64_t m_max_files = 0; uint64_t m_max_size = 5ULL * 1024 * 1024 * 1024; std::string m_msvc_dep_prefix = "Note: including file:"; @@ -411,6 +416,12 @@ Config::keep_comments_cpp() const return m_keep_comments_cpp; } +inline const std::vector& +Config::libexec_dirs() const +{ + return m_libexec_dirs; +} + inline const std::filesystem::path& Config::log_file() const { diff --git a/src/ccache/context.cpp b/src/ccache/context.cpp index ec82d17d1..adcf9f456 100644 --- a/src/ccache/context.cpp +++ b/src/ccache/context.cpp @@ -39,10 +39,10 @@ namespace fs = util::filesystem; -Context::Context() +Context::Context(const fs::path& ccache_exe_dir) : actual_cwd(fs::current_path().value_or("")), apparent_cwd(util::apparent_cwd(actual_cwd)), - storage(config), + storage(config, ccache_exe_dir), #ifdef INODE_CACHE_SUPPORTED inode_cache(config), #endif diff --git a/src/ccache/context.hpp b/src/ccache/context.hpp index be9b8e9dd..ea2af640a 100644 --- a/src/ccache/context.hpp +++ b/src/ccache/context.hpp @@ -47,7 +47,7 @@ class SignalHandler; class Context : util::NonCopyable { public: - Context(); + Context(const std::filesystem::path& ccache_exe_dir = {}); ~Context(); // Read configuration, initialize logging, etc. Typically not called from unit diff --git a/src/ccache/core/mainoptions.cpp b/src/ccache/core/mainoptions.cpp index 65b308936..c0684d06a 100644 --- a/src/ccache/core/mainoptions.cpp +++ b/src/ccache/core/mainoptions.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2021-2025 Joel Rosdahl and other contributors +// Copyright (C) 2021-2026 Joel Rosdahl and other contributors // // See doc/authors.adoc for a complete list of contributors. // @@ -171,6 +171,7 @@ Options for scripting or debugging: PATH --inspect PATH print result/manifest file at PATH in human-readable format + --stop-storage-helpers stop running storage helper(s), if any --print-log-stats print statistics counter IDs and corresponding values from the stats log in machine-parsable format @@ -449,6 +450,7 @@ enum : uint8_t { PRINT_STATS, PRINT_VERSION, SHOW_LOG_STATS, + STOP_STORAGE_HELPERS, THREADS, TRIM_DIR, TRIM_MAX_SIZE, @@ -461,44 +463,45 @@ enum : uint8_t { const char options_string[] = "cCd:k:hF:M:po:svVxX:z"; const option long_options[] = { - {"checksum-file", REQUIRED, nullptr, CHECKSUM_FILE }, - {"cleanup", NO_ARGUMENT, nullptr, 'c' }, - {"clear", NO_ARGUMENT, nullptr, 'C' }, - {"config-path", REQUIRED, nullptr, CONFIG_PATH }, - {"dir", REQUIRED, nullptr, 'd' }, - {"directory", REQUIRED, nullptr, 'd' }, // compat - {"dump-manifest", REQUIRED, nullptr, INSPECT }, // compat - {"dump-result", REQUIRED, nullptr, INSPECT }, // compat - {"evict-namespace", REQUIRED, nullptr, EVICT_NAMESPACE }, - {"evict-older-than", REQUIRED, nullptr, EVICT_OLDER_THAN}, - {"extract-result", REQUIRED, nullptr, EXTRACT_RESULT }, - {"format", REQUIRED, nullptr, FORMAT }, - {"get-config", REQUIRED, nullptr, 'k' }, - {"hash-file", REQUIRED, nullptr, HASH_FILE }, - {"help", NO_ARGUMENT, nullptr, 'h' }, - {"inspect", REQUIRED, nullptr, INSPECT }, - {"max-files", REQUIRED, nullptr, 'F' }, - {"max-size", REQUIRED, nullptr, 'M' }, - {"print-log-stats", NO_ARGUMENT, nullptr, PRINT_LOG_STATS }, - {"print-stats", NO_ARGUMENT, nullptr, PRINT_STATS }, - {"print-version", NO_ARGUMENT, nullptr, PRINT_VERSION }, - {"recompress", REQUIRED, nullptr, 'X' }, - {"recompress-threads", REQUIRED, nullptr, THREADS }, // compat - {"set-config", REQUIRED, nullptr, 'o' }, - {"show-compression", NO_ARGUMENT, nullptr, 'x' }, - {"show-config", NO_ARGUMENT, nullptr, 'p' }, - {"show-log-stats", NO_ARGUMENT, nullptr, SHOW_LOG_STATS }, - {"show-stats", NO_ARGUMENT, nullptr, 's' }, - {"threads", REQUIRED, nullptr, THREADS }, - {"trim-dir", REQUIRED, nullptr, TRIM_DIR }, - {"trim-max-size", REQUIRED, nullptr, TRIM_MAX_SIZE }, - {"trim-method", REQUIRED, nullptr, TRIM_METHOD }, - {"trim-recompress", REQUIRED, nullptr, TRIM_RECOMPRESS }, - {"trim-recompress-threads", REQUIRED, nullptr, THREADS }, // compat - {"verbose", NO_ARGUMENT, nullptr, 'v' }, - {"version", NO_ARGUMENT, nullptr, 'V' }, - {"zero-stats", NO_ARGUMENT, nullptr, 'z' }, - {nullptr, 0, nullptr, 0 } + {"checksum-file", REQUIRED, nullptr, CHECKSUM_FILE }, + {"cleanup", NO_ARGUMENT, nullptr, 'c' }, + {"clear", NO_ARGUMENT, nullptr, 'C' }, + {"config-path", REQUIRED, nullptr, CONFIG_PATH }, + {"dir", REQUIRED, nullptr, 'd' }, + {"directory", REQUIRED, nullptr, 'd' }, // compat + {"dump-manifest", REQUIRED, nullptr, INSPECT }, // compat + {"dump-result", REQUIRED, nullptr, INSPECT }, // compat + {"evict-namespace", REQUIRED, nullptr, EVICT_NAMESPACE }, + {"evict-older-than", REQUIRED, nullptr, EVICT_OLDER_THAN }, + {"extract-result", REQUIRED, nullptr, EXTRACT_RESULT }, + {"format", REQUIRED, nullptr, FORMAT }, + {"get-config", REQUIRED, nullptr, 'k' }, + {"hash-file", REQUIRED, nullptr, HASH_FILE }, + {"help", NO_ARGUMENT, nullptr, 'h' }, + {"inspect", REQUIRED, nullptr, INSPECT }, + {"max-files", REQUIRED, nullptr, 'F' }, + {"max-size", REQUIRED, nullptr, 'M' }, + {"print-log-stats", NO_ARGUMENT, nullptr, PRINT_LOG_STATS }, + {"print-stats", NO_ARGUMENT, nullptr, PRINT_STATS }, + {"print-version", NO_ARGUMENT, nullptr, PRINT_VERSION }, + {"recompress", REQUIRED, nullptr, 'X' }, + {"recompress-threads", REQUIRED, nullptr, THREADS }, // compat + {"set-config", REQUIRED, nullptr, 'o' }, + {"show-compression", NO_ARGUMENT, nullptr, 'x' }, + {"show-config", NO_ARGUMENT, nullptr, 'p' }, + {"show-log-stats", NO_ARGUMENT, nullptr, SHOW_LOG_STATS }, + {"show-stats", NO_ARGUMENT, nullptr, 's' }, + {"stop-storage-helpers", NO_ARGUMENT, nullptr, STOP_STORAGE_HELPERS}, + {"threads", REQUIRED, nullptr, THREADS }, + {"trim-dir", REQUIRED, nullptr, TRIM_DIR }, + {"trim-max-size", REQUIRED, nullptr, TRIM_MAX_SIZE }, + {"trim-method", REQUIRED, nullptr, TRIM_METHOD }, + {"trim-recompress", REQUIRED, nullptr, TRIM_RECOMPRESS }, + {"trim-recompress-threads", REQUIRED, nullptr, THREADS }, // compat + {"verbose", NO_ARGUMENT, nullptr, 'v' }, + {"version", NO_ARGUMENT, nullptr, 'V' }, + {"zero-stats", NO_ARGUMENT, nullptr, 'z' }, + {nullptr, 0, nullptr, 0 } }; int @@ -796,6 +799,12 @@ process_main_options(int argc, const char* const* argv) break; } + case STOP_STORAGE_HELPERS: { + storage::Storage storage(config, fs::path(argv[0]).parent_path()); + storage.stop_remote_storage_helpers(); + break; + } + case TRIM_DIR: if (!trim_max_size) { throw Error("please specify --trim-max-size when using --trim-dir"); diff --git a/src/ccache/storage/remote/CMakeLists.txt b/src/ccache/storage/remote/CMakeLists.txt index 8a14b7441..7d75cbf26 100644 --- a/src/ccache/storage/remote/CMakeLists.txt +++ b/src/ccache/storage/remote/CMakeLists.txt @@ -2,6 +2,7 @@ set( sources client.cpp filestorage.cpp + helper.cpp remotestorage.cpp ) diff --git a/src/ccache/storage/remote/filestorage.cpp b/src/ccache/storage/remote/filestorage.cpp index 1f34b6e44..8d8fc2b9b 100644 --- a/src/ccache/storage/remote/filestorage.cpp +++ b/src/ccache/storage/remote/filestorage.cpp @@ -108,7 +108,7 @@ FileStorageBackend::FileStorageBackend( util::value_or_throw(util::parse_umask(attr.value)); } else if (attr.key == "update-mtime") { m_update_mtime = attr.value == "true"; - } else if (!is_framework_attribute(attr.key)) { + } else { LOG("Unknown attribute: {}", attr.key); } } diff --git a/src/ccache/storage/remote/helper.cpp b/src/ccache/storage/remote/helper.cpp new file mode 100644 index 000000000..fbdd90d05 --- /dev/null +++ b/src/ccache/storage/remote/helper.cpp @@ -0,0 +1,654 @@ +// Copyright (C) 2025-2026 Joel Rosdahl and other contributors +// +// See doc/authors.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "helper.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef _WIN32 +# include +# include +# include +# include +# include +#endif + +#include + +#ifndef environ +DLLIMPORT extern char** environ; +#endif + +using namespace std::chrono_literals; + +namespace fs = util::filesystem; + +namespace storage::remote { + +namespace { + +#ifdef _WIN32 +constexpr std::string_view k_named_pipe_prefix = "\\\\.\\pipe\\"; +#endif + +// Call a function that returns 0 on success and either an error code or -1 (and +// sets errno) on failure. +#define CHECK_LIB_CALL(function, ...) \ + do { \ + int _result = function(__VA_ARGS__); \ + if (_result != 0) { \ + return tl::unexpected(FMT(#function " failed: {}", \ + strerror(_result == -1 ? errno : _result))); \ + } \ + } while (false) + +// Generate a user-specific, unique socket/pipe path based on URL and +// attributes. +std::string +generate_endpoint_name( + const Url& url, + const std::vector& attributes) +{ + static const uint8_t delimiter[1] = {0}; + + Hash hash; +#ifdef _WIN32 + char username[256]; + DWORD username_len = sizeof(username); + if (GetUserNameA(username, &username_len)) { + hash.hash(username); + } +#else + hash.hash(static_cast(getuid())); +#endif + hash.hash(delimiter); + hash.hash(url.str()); + for (const auto& attr : attributes) { + hash.hash(delimiter); + hash.hash(attr.key); + hash.hash(delimiter); + hash.hash(attr.value); + } + return FMT("storage-{}-{}", url.scheme(), util::format_base16(hash.digest())); +} + +#ifndef _WIN32 +// Choose a short and safe base directory for Unix socket. +// +// Rationale: +// - Unix socket paths have a strict length limit (sun_path). +// - The ccache configured temporary dir can become very long (e.g. in CI). +// - We want a directory that is private to the user to avoid other users +// squatting the socket name. +std::optional +get_helper_ipc_dir() +{ + // If XDG_RUNTIME_DIR is set, use the same location as + // Config::default_temporary_dir. + if (const auto dir = Config::get_xdg_runtime_tmp_dir(); !dir.empty()) { + return dir; + } + + // Otherwise, create a per-user private directory under /tmp. We intentionally + // use /tmp instead of $TMPDIR to keep socket paths short. + fs::path dir = FMT("/tmp/ccache-tmp-{}", static_cast(getuid())); + if (auto r = fs::create_directories(dir); !r) { + LOG("Failed to create helper IPC dir {}: {}", dir, r.error()); + return std::nullopt; + } + + // Ensure correct permissions regardless of umask. + if (chmod(dir.c_str(), 0700) != 0) { + LOG("Failed to chmod helper IPC dir {}: {}", dir, strerror(errno)); + return std::nullopt; + } + + util::DirEntry entry(dir); + if (!entry) { + LOG("Failed to stat helper IPC dir {}: {}", + dir, + strerror(entry.error_number())); + return std::nullopt; + } + if (!entry.is_directory() || entry.is_symlink()) { + LOG("Helper IPC dir {} is not a directory", dir); + return std::nullopt; + } + if ((entry.mode() & 0077) != 0) { + LOG("Helper IPC dir {} is not private (mode {:o})", dir, entry.mode()); + return std::nullopt; + } + + return dir; +} +#endif + +std::vector +build_helper_env( + const Url& url, + std::string_view ipc_endpoint, + std::chrono::milliseconds idle_timeout, + const std::vector& attributes) +{ + std::vector env_vars; + env_vars.emplace_back(FMT("CRSH_IPC_ENDPOINT={}", ipc_endpoint)); + env_vars.emplace_back(FMT("CRSH_URL={}", url.str())); + env_vars.emplace_back( + FMT("CRSH_IDLE_TIMEOUT={}", idle_timeout.count() / 1000)); + env_vars.emplace_back(FMT("CRSH_NUM_ATTR={}", attributes.size())); + + for (size_t i = 0; i < attributes.size(); ++i) { + env_vars.emplace_back(FMT("CRSH_ATTR_KEY_{}={}", i, attributes[i].key)); + env_vars.emplace_back(FMT("CRSH_ATTR_VALUE_{}={}", i, attributes[i].value)); + } + + return env_vars; +} + +bool +is_ccache_crsh_var(std::string_view entry) +{ + auto [name, value] = util::split_once(entry, '='); + if (!value) { + return false; + } + + return name == "CRSH_IPC_ENDPOINT" || name == "CRSH_URL" + || name == "CRSH_IDLE_TIMEOUT" || name == "CRSH_NUM_ATTR" + || util::starts_with(name, "CRSH_ATTR_KEY_") + || util::starts_with(name, "CRSH_ATTR_VALUE_"); +} + +#ifndef _WIN32 +std::vector +get_fds_to_close() +{ + std::vector fds_to_close; + +# ifdef __linux__ + // Enumerate open FDs via /proc/self/fd for efficiency. (Using opendir instead + // of std::filesystem to be able to filter out dir_fd.) + bool enumerated = false; + if (DIR* dir = opendir("/proc/self/fd"); dir) { + enumerated = true; + const int dir_fd = dirfd(dir); + struct dirent* entry; + while ((entry = readdir(dir)) != nullptr) { + if (entry->d_name[0] == '.') { + continue; // skip . and .. + } + char* endptr; + long fd = std::strtol(entry->d_name, &endptr, 10); + if (*endptr == '\0' && fd >= 3 && fd != dir_fd) { + fds_to_close.push_back(static_cast(fd)); + } + } + closedir(dir); + } + + if (enumerated) { + return fds_to_close; + } +# endif + + // Fallback: check FDs up to a reasonable limit. + long max_fd = sysconf(_SC_OPEN_MAX); + if (max_fd < 0 || max_fd > 1024) { + max_fd = 1024; // cap to avoid thousands of fcntl syscalls + } + for (int fd = 3; fd < max_fd; ++fd) { + // We must verify that the FD exists because on some systems (e.g. macOS), + // posix_spawn will fail when trying to close a non-existent FD. + if (fcntl(fd, F_GETFD) != -1) { + fds_to_close.push_back(fd); + } + } + + return fds_to_close; +} +#endif + +tl::expected +spawn_helper(const fs::path& helper_path, + std::string_view endpoint, + const Url& url, + std::chrono::milliseconds idle_timeout, + const std::vector& attributes) +{ + LOG("Spawning storage helper {} for {}", helper_path, endpoint); + +#ifdef _WIN32 + // Don't pass \\.\pipe\ prefix on Windows. + DEBUG_ASSERT(util::starts_with(endpoint, k_named_pipe_prefix)); + auto ipc_endpoint = endpoint.substr(k_named_pipe_prefix.length()); +#else + const auto& ipc_endpoint = endpoint; +#endif + + const auto env_vars = + build_helper_env(url, ipc_endpoint, idle_timeout, attributes); + +#ifdef _WIN32 + std::string env_block; + for (char** env = environ; *env != nullptr; ++env) { + if (!is_ccache_crsh_var(*env)) { + env_block += *env; + env_block += '\0'; + } + } + for (const auto& var : env_vars) { + env_block += var; + env_block += '\0'; + } + env_block += '\0'; + + STARTUPINFO si; + PROCESS_INFORMATION pi; + ZeroMemory(&si, sizeof(si)); + si.cb = sizeof(si); + ZeroMemory(&pi, sizeof(pi)); + + std::string application = helper_path.string(); + // CreateProcess may write into lpCommandLine, so it should be a mutable + // buffer != application. + std::string cmdline = application; + if (!CreateProcess(application.c_str(), + cmdline.data(), + nullptr, + nullptr, + FALSE, + CREATE_NO_WINDOW | DETACHED_PROCESS, + env_block.data(), + nullptr, + &si, + &pi)) { + DWORD error = GetLastError(); + return tl::unexpected( + FMT("{} ({})", util::win32_error_message(error), error)); + } + + CloseHandle(pi.hThread); + CloseHandle(pi.hProcess); +#else + std::vector env_strings; + for (char** env = environ; *env != nullptr; ++env) { + if (!is_ccache_crsh_var(*env)) { + env_strings.emplace_back(*env); + } + } + env_strings.insert(env_strings.end(), env_vars.begin(), env_vars.end()); + + std::vector env_ptrs; + env_ptrs.reserve(env_strings.size() + 1); + for (auto& str : env_strings) { + env_ptrs.push_back(const_cast(str.c_str())); + } + env_ptrs.push_back(nullptr); + + posix_spawn_file_actions_t actions; + CHECK_LIB_CALL(posix_spawn_file_actions_init, &actions); + DEFER(posix_spawn_file_actions_destroy(&actions)); + + CHECK_LIB_CALL( + posix_spawn_file_actions_addopen, &actions, 0, "/dev/null", O_RDONLY, 0); + CHECK_LIB_CALL( + posix_spawn_file_actions_addopen, &actions, 1, "/dev/null", O_WRONLY, 0); + CHECK_LIB_CALL( + posix_spawn_file_actions_addopen, &actions, 2, "/dev/null", O_WRONLY, 0); + + // We need to close all inherited FDs since keeping them open in the + // long-lived helper process can interfere with build systems, see for example + // . + for (int fd : get_fds_to_close()) { + CHECK_LIB_CALL(posix_spawn_file_actions_addclose, &actions, fd); + } + + posix_spawnattr_t attr; + CHECK_LIB_CALL(posix_spawnattr_init, &attr); + DEFER(posix_spawnattr_destroy(&attr)); + + // Create a new session to fully detach from the controlling terminal. +# ifdef POSIX_SPAWN_SETSID + CHECK_LIB_CALL(posix_spawnattr_setflags, &attr, POSIX_SPAWN_SETSID); +# else + // Fallback for systems without POSIX_SPAWN_SETSID. + CHECK_LIB_CALL(posix_spawnattr_setflags, &attr, POSIX_SPAWN_SETPGROUP); + CHECK_LIB_CALL(posix_spawnattr_setpgroup, &attr, 0); +# endif + + pid_t pid; + char* argv[] = {const_cast(helper_path.c_str()), nullptr}; + int result = posix_spawnp( + &pid, helper_path.c_str(), &actions, &attr, argv, env_ptrs.data()); + + if (result != 0) { + return tl::unexpected(strerror(result)); + } + + LOG("Spawned helper process with PID {}", pid); +#endif + + return {}; +} + +// Backend implementation that communicates with a helper process. +class HelperBackend : public RemoteStorage::Backend +{ +public: + HelperBackend(const fs::path& helper_path, + const fs::path& temp_dir, + const Url& url, + const std::vector& attributes, + std::chrono::milliseconds data_timeout, + std::chrono::milliseconds request_timeout, + std::chrono::milliseconds idle_timeout); + + tl::expected, Failure> + get(const Hash::Digest& key) override; + + tl::expected put(const Hash::Digest& key, + nonstd::span value, + Overwrite overwrite) override; + + tl::expected remove(const Hash::Digest& key) override; + + void stop() override; + +private: + fs::path m_helper_path; + std::string m_endpoint; // Unix socket on POSIX, pipe name on Windows + fs::path m_endpoint_lock_path; // path to lock for guarding spawn of helper + Url m_url; + std::vector m_attributes; + std::chrono::milliseconds m_idle_timeout; + Client m_client; + bool m_connected = false; + + tl::expected ensure_connected(bool spawn = true); + tl::expected finalize_connection(); +}; + +HelperBackend::HelperBackend(const fs::path& helper_path, + const fs::path& temp_dir, + const Url& url, + const std::vector& attributes, + std::chrono::milliseconds data_timeout, + std::chrono::milliseconds request_timeout, + std::chrono::milliseconds idle_timeout) + : m_helper_path(helper_path), + m_url(url), + m_attributes(attributes), + m_idle_timeout(idle_timeout), + m_client(data_timeout, request_timeout) +{ + if (m_helper_path.empty()) { + // The "crsh:" URL case: +#ifdef _WIN32 + m_endpoint = FMT("{}{}", k_named_pipe_prefix, url.path()); +#else + m_endpoint = url.path(); +#endif + // No m_endpoint_lock_path needed since we won't spawn a helper. + } else { + // The common case: + auto endpoint_name = generate_endpoint_name(url, attributes); +#ifdef _WIN32 + m_endpoint = FMT("{}ccache-{}", k_named_pipe_prefix, endpoint_name); + m_endpoint_lock_path = FMT("{}/{}", temp_dir, endpoint_name); +#else + auto helper_ipc_dir = get_helper_ipc_dir(); + if (!helper_ipc_dir) { + LOG("Failed to select helper IPC dir, falling back to {}", temp_dir); + helper_ipc_dir = temp_dir; + } + m_endpoint = FMT("{}/{}", *helper_ipc_dir, endpoint_name); + m_endpoint_lock_path = m_endpoint; +#endif + } +} + +tl::expected +HelperBackend::finalize_connection() +{ + if (m_client.protocol_version() != Client::k_protocol_version) { + LOG("Unexpected remote storage helper protocol version: {} (!= {})", + m_client.protocol_version(), + Client::k_protocol_version); + return tl::unexpected(Failure::error); + } + + if (!m_client.has_capability(Client::Capability::get_put_remove_stop)) { + LOG_RAW("Remote storage helper does not support capability 0"); + return tl::unexpected(Failure::error); + } + + m_connected = true; + return {}; +} + +tl::expected +HelperBackend::ensure_connected(bool spawn) +{ + if (m_connected) { + return {}; + } + + // Try to connect to an existing helper. + util::Timer timer; + auto connect_result = m_client.connect(m_endpoint); + if (connect_result) { + LOG("Connected to existing remote storage helper at {} ({:.2f} ms)", + m_endpoint, + timer.measure_ms()); + return finalize_connection(); + } + LOG( + "Failed to connect to existing remote storage helper at {}: {} ({:.2f} ms)", + m_endpoint, + connect_result.error().message, + timer.measure_ms()); + + if (!spawn) { + return {}; + } + + if (m_helper_path.empty()) { + // Could not connect to "crsh:" endpoint, so just fail. + return tl::unexpected(Failure::error); + } + + // No existing helper, spawn a new one. Use a lock file to prevent multiple + // processes from spawning simultaneously. + util::LockFile spawn_lock(m_endpoint_lock_path); + if (!spawn_lock.acquire()) { + LOG_RAW("Failed to acquire spawn lock"); + return tl::unexpected(Failure::error); + } + + // We have the lock. Check again if another process spawned while we waited. + timer.reset(); + if (m_client.connect(m_endpoint)) { + LOG( + "Connected to remote storage helper spawned by another process ({:.2f}" + " ms)", + timer.measure_ms()); + return finalize_connection(); + } + + // No helper exists, spawn it now. + timer.reset(); + auto spawn_result = spawn_helper( + m_helper_path, m_endpoint, m_url, m_idle_timeout, m_attributes); + if (!spawn_result) { + LOG("Failed to spawn helper: {}", spawn_result.error()); + return tl::unexpected(Failure::error); + } + LOG("Spawned remote storage helper ({:.2f} ms)", timer.measure_ms()); + timer.reset(); + + constexpr auto sleep_duration = 1ms; + constexpr double spawn_timeout_ms = 1000.0; + + timer.reset(); + while (timer.measure_ms() < spawn_timeout_ms) { + connect_result = m_client.connect(m_endpoint); + if (connect_result) { + LOG("Connected to newly spawned remote storage helper at {} ({:.2f} ms)", + m_endpoint, + timer.measure_ms()); + return finalize_connection(); + } + + std::this_thread::sleep_for(sleep_duration); + } + + LOG("Failed to connect to spawned remote storage helper: {}", + connect_result.error().message); + + return tl::unexpected(Failure::timeout); +} + +tl::expected, RemoteStorage::Backend::Failure> +HelperBackend::get(const Hash::Digest& key) +{ + TRY(ensure_connected()); + + auto result = m_client.get(key); + if (!result) { + const auto& error = result.error(); + LOG("Remote storage get failed: {}", error.message); + auto failure = (error.failure == Client::Failure::timeout) + ? Failure::timeout + : Failure::error; + return tl::unexpected(failure); + } + + return *result; +} + +tl::expected +HelperBackend::put(const Hash::Digest& key, + nonstd::span value, + Overwrite overwrite) +{ + TRY(ensure_connected()); + + Client::PutFlags flags; + flags.overwrite = (overwrite == Overwrite::yes); + + auto result = m_client.put(key, value, flags); + if (!result) { + const auto& error = result.error(); + LOG("Remote storage put failed: {}", error.message); + auto failure = (error.failure == Client::Failure::timeout) + ? Failure::timeout + : Failure::error; + return tl::unexpected(failure); + } + + return *result; +} + +tl::expected +HelperBackend::remove(const Hash::Digest& key) +{ + TRY(ensure_connected()); + + auto result = m_client.remove(key); + if (!result) { + const auto& error = result.error(); + LOG("Remote storage remove failed: {}", error.message); + auto failure = (error.failure == Client::Failure::timeout) + ? Failure::timeout + : Failure::error; + return tl::unexpected(failure); + } + + return *result; +} + +void +HelperBackend::stop() +{ + if (auto r = ensure_connected(false); !r) { + LOG_RAW("Failed to connect to remote storage helper"); + return; + } + if (!m_connected) { + LOG("No need to stop remote storage helper for {}", m_url.str()); + return; + } + if (auto r = m_client.stop(); r) { + LOG("Stopped remote storage helper for {}", m_url.str()); + } else { + LOG("Failed to stop remote storage helper for {}: {}", + m_url.str(), + r.error().message); + } +} + +} // namespace + +Helper::Helper(const std::filesystem::path& helper_path, + const fs::path& temp_dir, + std::chrono::milliseconds data_timeout, + std::chrono::milliseconds request_timeout, + std::chrono::milliseconds idle_timeout) + : m_helper_path(helper_path), + m_temp_dir(temp_dir), + m_data_timeout(data_timeout), + m_request_timeout(request_timeout), + m_idle_timeout(idle_timeout) +{ +} + +Helper::Helper(std::chrono::milliseconds data_timeout, + std::chrono::milliseconds request_timeout) + : m_data_timeout(data_timeout), + m_request_timeout(request_timeout) +{ +} + +std::unique_ptr +Helper::create_backend(const Url& url, + const std::vector& attributes) const +{ + return std::make_unique(m_helper_path, + m_temp_dir, + url, + attributes, + m_data_timeout, + m_request_timeout, + m_idle_timeout); +} + +} // namespace storage::remote diff --git a/src/ccache/storage/remote/helper.hpp b/src/ccache/storage/remote/helper.hpp new file mode 100644 index 000000000..1d6cfe2d4 --- /dev/null +++ b/src/ccache/storage/remote/helper.hpp @@ -0,0 +1,55 @@ +// Copyright (C) 2025-2026 Joel Rosdahl and other contributors +// +// See doc/authors.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include + +#include +#include +#include +#include + +namespace storage::remote { + +class Helper : public RemoteStorage +{ +public: + Helper(const std::filesystem::path& helper_path, + const std::filesystem::path& temp_dir, + std::chrono::milliseconds data_timeout, + std::chrono::milliseconds request_timeout, + std::chrono::milliseconds idle_timeout); + + // Special case: crsh scheme + Helper(std::chrono::milliseconds data_timeout, + std::chrono::milliseconds request_timeout); + + std::unique_ptr create_backend( + const Url& url, + const std::vector& attributes) const override; + +private: + std::filesystem::path m_helper_path; // empty -> connect to existing socket + std::filesystem::path m_temp_dir; + std::chrono::milliseconds m_data_timeout; + std::chrono::milliseconds m_request_timeout; + std::chrono::milliseconds m_idle_timeout; +}; + +} // namespace storage::remote diff --git a/src/ccache/storage/remote/httpstorage.cpp b/src/ccache/storage/remote/httpstorage.cpp index 5f7d8719a..99dac9d90 100644 --- a/src/ccache/storage/remote/httpstorage.cpp +++ b/src/ccache/storage/remote/httpstorage.cpp @@ -158,7 +158,7 @@ HttpStorageBackend::HttpStorageBackend( } else { LOG("Incomplete header specification: {}", attr.value); } - } else if (!is_framework_attribute(attr.key)) { + } else { LOG("Unknown attribute: {}", attr.key); } } @@ -312,17 +312,4 @@ HttpStorage::create_backend( return std::make_unique(url, attributes); } -void -HttpStorage::redact_secrets(std::vector& attributes) const -{ - auto bearer_token_attribute = - std::find_if(attributes.begin(), attributes.end(), [&](const auto& attr) { - return attr.key == "bearer-token"; - }); - if (bearer_token_attribute != attributes.end()) { - bearer_token_attribute->value = storage::k_redacted_secret; - bearer_token_attribute->raw_value = storage::k_redacted_secret; - } -} - } // namespace storage::remote diff --git a/src/ccache/storage/remote/httpstorage.hpp b/src/ccache/storage/remote/httpstorage.hpp index 371253bf8..4866f3ff0 100644 --- a/src/ccache/storage/remote/httpstorage.hpp +++ b/src/ccache/storage/remote/httpstorage.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2021-2024 Joel Rosdahl and other contributors +// Copyright (C) 2021-2026 Joel Rosdahl and other contributors // // See doc/authors.adoc for a complete list of contributors. // @@ -33,9 +33,6 @@ public: std::unique_ptr create_backend( const Url& url, const std::vector& attributes) const override; - - void - redact_secrets(std::vector& attributes) const override; }; } // namespace storage::remote diff --git a/src/ccache/storage/remote/redisstorage.cpp b/src/ccache/storage/remote/redisstorage.cpp index f7515a5bb..a2aa727a8 100644 --- a/src/ccache/storage/remote/redisstorage.cpp +++ b/src/ccache/storage/remote/redisstorage.cpp @@ -138,7 +138,7 @@ RedisStorageBackend::RedisStorageBackend( connect_timeout = parse_timeout_attribute(attr.value); } else if (attr.key == "operation-timeout") { operation_timeout = parse_timeout_attribute(attr.value); - } else if (!is_framework_attribute(attr.key)) { + } else { LOG("Unknown attribute: {}", attr.key); } } diff --git a/src/ccache/storage/remote/remotestorage.cpp b/src/ccache/storage/remote/remotestorage.cpp index 29b563106..9d4b10c37 100644 --- a/src/ccache/storage/remote/remotestorage.cpp +++ b/src/ccache/storage/remote/remotestorage.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2021-2024 Joel Rosdahl and other contributors +// Copyright (C) 2021-2026 Joel Rosdahl and other contributors // // See doc/authors.adoc for a complete list of contributors. // @@ -23,12 +23,6 @@ namespace storage::remote { -bool -RemoteStorage::Backend::is_framework_attribute(const std::string& name) -{ - return name == "read-only" || name == "shards"; -} - std::chrono::milliseconds RemoteStorage::Backend::parse_timeout_attribute(const std::string& value) { diff --git a/src/ccache/storage/remote/remotestorage.hpp b/src/ccache/storage/remote/remotestorage.hpp index 8b3868018..22bb46138 100644 --- a/src/ccache/storage/remote/remotestorage.hpp +++ b/src/ccache/storage/remote/remotestorage.hpp @@ -38,6 +38,10 @@ namespace storage::remote { const auto k_default_connect_timeout = std::chrono::milliseconds{100}; const auto k_default_operation_timeout = std::chrono::milliseconds{10000}; +const auto k_default_data_timeout = std::chrono::seconds{1}; +const auto k_default_request_timeout = std::chrono::seconds{10}; +const auto k_default_idle_timeout = std::chrono::minutes{10}; + // This class defines the API that a remote storage must implement. class RemoteStorage { @@ -86,9 +90,8 @@ public: // removed, otherwise false. virtual tl::expected remove(const Hash::Digest& key) = 0; - // Determine whether an attribute is handled by the remote storage - // framework itself. - static bool is_framework_attribute(const std::string& name); + // Stop backend. + virtual void stop(); // Parse a timeout `value`, throwing `Failed` on error. static std::chrono::milliseconds @@ -105,20 +108,10 @@ public: virtual std::unique_ptr create_backend(const Url& url, const std::vector& attributes) const = 0; - - // Redact secrets in backend attributes, if any. - virtual void - redact_secrets(std::vector& attributes) const; }; // --- Inline implementations --- -inline void -RemoteStorage::redact_secrets( - std::vector& /*attributes*/) const -{ -} - inline RemoteStorage::Backend::Failed::Failed(Failure failure) : Failed("", failure) { @@ -137,4 +130,9 @@ RemoteStorage::Backend::Failed::failure() const return m_failure; } +inline void +RemoteStorage::Backend::stop() +{ +} + } // namespace storage::remote diff --git a/src/ccache/storage/storage.cpp b/src/ccache/storage/storage.cpp index 6020b99cb..320874f57 100644 --- a/src/ccache/storage/storage.cpp +++ b/src/ccache/storage/storage.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2021-2025 Joel Rosdahl and other contributors +// Copyright (C) 2021-2026 Joel Rosdahl and other contributors // // See doc/authors.adoc for a complete list of contributors. // @@ -19,20 +19,26 @@ #include "storage.hpp" #include +#include #include #include #include #include +#include #ifdef HAVE_HTTP_STORAGE_BACKEND # include #endif #ifdef HAVE_REDIS_STORAGE_BACKEND # include #endif +#include #include #include +#include +#include #include #include +#include #include #include #include @@ -48,11 +54,15 @@ #include #include +using namespace std::chrono_literals; + +namespace fs = util::filesystem; + namespace storage { -const std::unordered_map> - k_remote_storage_implementations = { + k_builtin_remote_storage_implementations = { {"file", std::make_shared() }, #ifdef HAVE_HTTP_STORAGE_BACKEND {"http", std::make_shared() }, @@ -67,11 +77,13 @@ std::vector get_features() { std::vector features; - features.reserve(k_remote_storage_implementations.size()); - std::transform(k_remote_storage_implementations.begin(), - k_remote_storage_implementations.end(), + features.reserve(k_builtin_remote_storage_implementations.size()); + std::transform(k_builtin_remote_storage_implementations.begin(), + k_builtin_remote_storage_implementations.end(), std::back_inserter(features), [](auto& entry) { return FMT("{}-storage", entry.first); }); + features.emplace_back("crsh-storage"); + features.emplace_back("remote-storage"); return features; } @@ -86,16 +98,16 @@ struct RemoteStorageShardConfig // Representation of one entry in the remote_storage config option. struct RemoteStorageConfig { - // Raw URL with unexpanded "*". - std::string url_str; - - // "shard" attribute. - std::vector shards; + std::string url_str; // Raw URL with unexpanded "*" - // "read-only" attribute. - bool read_only = false; + std::optional data_timeout; // "data-timeout" + std::string helper; // "helper" + std::optional idle_timeout; // "idle-timeout" + std::optional read_only; // "read-only" + std::optional request_timeout; // "request-timeout" + std::vector shards; // "shards" - // Other attributes. + // Custom attributes (@key=value). std::vector attributes; }; @@ -117,12 +129,44 @@ struct RemoteStorageEntry }; static std::string -to_string(const RemoteStorageConfig& entry) +to_string(const storage::RemoteStorageConfig& entry) { std::string result = entry.url_str; + + if (entry.data_timeout) { + result += + FMT(" data-timeout={}", util::format_duration(*entry.data_timeout)); + } + if (!entry.helper.empty()) { + result += FMT(" helper={}", entry.helper); + } + if (entry.idle_timeout) { + result += + FMT(" idle-timeout={}", util::format_duration(*entry.idle_timeout)); + } + if (entry.read_only) { + result += FMT(" read-only={}", *entry.read_only ? "true" : "false"); + } + if (entry.request_timeout) { + result += + FMT(" request-timeout={}", util::format_duration(*entry.request_timeout)); + } + if (!entry.shards.empty() && !entry.shards[0].name.empty()) { + result += " shards="; + bool first = true; + for (auto shard : entry.shards) { + if (first) { + first = false; + } else { + result += ','; + } + result += FMT("{}({})", shard.name, shard.weight); + } + } for (const auto& attr : entry.attributes) { - result += FMT("|{}={}", attr.key, attr.raw_value); + result += FMT(" @{}={}", attr.key, attr.raw_value); } + return result; } @@ -143,80 +187,123 @@ url_from_string(const std::string& url_string) return url; } -static RemoteStorageConfig -parse_storage_config(const std::string_view entry) +static std::vector +parse_shards(std::string_view url_str, std::string_view value) { - const auto parts = - util::split_into_views(entry, "|", util::Tokenizer::Mode::include_empty); + std::vector result; - if (parts.empty() || parts.front().empty()) { + const auto asterisk_count = std::count(url_str.begin(), url_str.end(), '*'); + if (asterisk_count == 0) { throw core::Error( - FMT("remote storage config must provide a URL: {}", entry)); + FMT(R"(Missing "*" in URL when using shards: "{}")", url_str)); + } else if (asterisk_count > 1) { + throw core::Error( + FMT(R"(Multiple "*" in URL when using shards: "{}")", url_str)); } + std::string scheme; + for (const auto& shard : util::Tokenizer(value, ",")) { + double weight = 1.0; + std::string_view name; + const auto lp_pos = shard.find('('); + if (lp_pos != std::string_view::npos) { + if (shard.back() != ')') { + throw core::Error(FMT("Invalid shard name: \"{}\"", shard)); + } + weight = util::value_or_throw(util::parse_double( + std::string(shard.substr(lp_pos + 1, shard.length() - lp_pos - 2)))); + if (weight < 0.0) { + throw core::Error(FMT("Invalid shard weight: \"{}\"", weight)); + } + name = shard.substr(0, lp_pos); + } else { + name = shard; + } + Url url = util::value_or_throw( + url_from_string(util::replace_first(url_str, "*", name))); + if (scheme.empty()) { + scheme = url.scheme(); + } else if (url.scheme() != scheme) { + throw core::Error( + FMT("Expanding {} with shards results in different schemes ({} != {})", + url_str, + url.scheme(), + scheme)); + } + result.push_back({std::string(name), weight, url}); + } + + return result; +} + +namespace { + +enum class ForLogging { yes, no }; + +} + +static RemoteStorageConfig +parse_storage_config(const std::vector::const_iterator& begin, + const std::vector::const_iterator& end, + ForLogging for_logging) +{ + ASSERT(begin != end); RemoteStorageConfig result; - result.url_str = std::string(parts[0]); - const auto& url_str = result.url_str; - for (size_t i = 1; i < parts.size(); ++i) { - if (parts[i].empty()) { - continue; - } - const auto [key, right_hand_side] = - util::split_once_into_views(parts[i], '='); - const auto& raw_value = right_hand_side.value_or("true"); + result.url_str = std::string(*begin); + auto& url_str = result.url_str; + + for (auto part = begin + 1; part != end; ++part) { + auto parts = util::split_once_into_views(*part, '='); + auto key = parts.first; + auto right = parts.second; + const auto& raw_value = right.value_or("true"); const auto value = util::value_or_throw(util::percent_decode(raw_value)); - if (key == "read-only") { + + std::array backward_compat_attributes = { + "bearer-token", + "connect-timeout", + "header", + "keep-alive", + "layout", + "operation-timeout", + "umask", + "update-mtime", + }; + std::string backward_compat_key; + if (std::any_of(backward_compat_attributes.begin(), + backward_compat_attributes.end(), + [&](const char* attr) { return key == attr; })) { + backward_compat_key = FMT("@{}", key); + key = backward_compat_key; + } + + if (!key.empty() && key.front() == '@') { + result.attributes.push_back( + {std::string(key.substr(1)), value, std::string(raw_value)}); + } else if (key == "data-timeout") { + result.data_timeout = + util::value_or_throw(util::parse_duration(value)); + } else if (key == "idle-timeout") { + result.idle_timeout = + util::value_or_throw(util::parse_duration(value)); + } else if (key == "helper") { + result.helper = value; + } else if (key == "read-only") { result.read_only = (value == "true"); + } else if (key == "request-timeout") { + result.request_timeout = + util::value_or_throw(util::parse_duration(value)); } else if (key == "shards") { - const auto asterisk_count = - std::count(url_str.begin(), url_str.end(), '*'); - if (asterisk_count == 0) { - throw core::Error( - FMT(R"(Missing "*" in URL when using shards: "{}")", url_str)); - } else if (asterisk_count > 1) { - throw core::Error( - FMT(R"(Multiple "*" in URL when using shards: "{}")", url_str)); - } - std::string scheme; - for (const auto& shard : util::Tokenizer(value, ",")) { - double weight = 1.0; - std::string_view name; - const auto lp_pos = shard.find('('); - if (lp_pos != std::string_view::npos) { - if (shard.back() != ')') { - throw core::Error(FMT("Invalid shard name: \"{}\"", shard)); - } - weight = - util::value_or_throw(util::parse_double(std::string( - shard.substr(lp_pos + 1, shard.length() - lp_pos - 2)))); - if (weight < 0.0) { - throw core::Error(FMT("Invalid shard weight: \"{}\"", weight)); - } - name = shard.substr(0, lp_pos); - } else { - name = shard; - } - - Url url = util::value_or_throw( - url_from_string(util::replace_first(url_str, "*", name))); - if (!scheme.empty() && url.scheme() != scheme) { - throw core::Error(FMT("Scheme {} different from {} in {}", - url.scheme(), - scheme, - url_str)); - } - result.shards.push_back({std::string(name), weight, url}); - } + result.shards = parse_shards(url_str, value); + } else if (for_logging == ForLogging::no) { + LOG("Unknown remote config property: {}", *part); } - - result.attributes.push_back( - {std::string(key), value, std::string(raw_value)}); } - // No shards => save the single URL as the sole shard. if (result.shards.empty()) { + // No shards => save the single URL as the sole shard. result.shards.push_back( {"", 0.0, util::value_or_throw(url_from_string(url_str))}); } @@ -225,24 +312,97 @@ parse_storage_config(const std::string_view entry) } static std::vector -parse_storage_configs(const std::string_view& configs) +parse_storage_configs(const std::string_view& config, ForLogging for_logging) { std::vector result; - for (const auto& config : util::Tokenizer(configs, " ")) { - result.push_back(parse_storage_config(config)); + + // Split on "|" as well for backward compatibility. + auto parts = util::split_into_views(config, " \t|"); + + auto it = parts.begin(); + while (it != parts.end()) { + // Find end of config entry, i.e. next URL or end of parts. + auto it2 = it + 1; + while (it2 != parts.end()) { + auto [left, right] = util::split_once_into_views(*it2, ':'); + if (right && left.find('=') == std::string_view::npos) { + // foo:... but not foo=...:... -> looks like a URL + break; + } + ++it2; + } + result.push_back(parse_storage_config(it, it2, for_logging)); + it = it2; } + return result; } +static fs::path +find_remote_storage_helper(std::string_view name, + const std::vector search_dirs) +{ + bool bare_helper_name = +#ifdef _WIN32 + name.find('\\') == std::string::npos && +#endif + name.find('/') == std::string::npos; + fs::path path = bare_helper_name ? find_executable_in_path(name, search_dirs) +#ifdef _WIN32 + : util::add_exe_suffix(name); +#else + : name; +#endif + + if (!path.empty() && fs::exists(path)) { + LOG("Found remote storage helper {}", path); + return path; + } + + LOG("Could not find remote storage helper program \"{}\"", name); + return {}; +} + static std::shared_ptr -get_storage(const std::string& scheme) +get_remote_storage(std::string_view scheme, + const RemoteStorageConfig& config, + const fs::path& temp_dir, + const std::vector& helper_search_dirs) { - const auto it = k_remote_storage_implementations.find(scheme); - if (it != k_remote_storage_implementations.end()) { - return it->second; + // Special case: crsh scheme connects directly to storage helper socket. + if (scheme == "crsh") { + return std::make_shared( + config.data_timeout.value_or(remote::k_default_data_timeout), + config.request_timeout.value_or(remote::k_default_request_timeout)); + } + + if (config.helper == "_builtin_" || scheme == "file") { + LOG_RAW("Forcing use of builtin storage backend"); } else { - return {}; + // Look up and use storage helper if available. + std::string helper_name = + !config.helper.empty() ? config.helper : FMT("ccache-storage-{}", scheme); + // TODO: If/when we in the future remove built-in http and redis backends, + // we can search for the helper program on demand instead. + fs::path helper_path = + find_remote_storage_helper(helper_name, helper_search_dirs); + if (!helper_path.empty()) { + return std::make_shared( + helper_path, + temp_dir, + config.data_timeout.value_or(remote::k_default_data_timeout), + config.request_timeout.value_or(remote::k_default_request_timeout), + config.idle_timeout.value_or(remote::k_default_idle_timeout)); + } + } + + // Fall back to builtin implementation. + const auto it = k_builtin_remote_storage_implementations.find(scheme); + if (it != k_builtin_remote_storage_implementations.end()) { + return it->second; } + + return {}; } std::string @@ -255,9 +415,10 @@ get_redacted_url_str_for_logging(const Url& url) return redacted_url.str(); } -Storage::Storage(const Config& config) +Storage::Storage(const Config& config, const fs::path& ccache_exe_dir) : local(config), - m_config(config) + m_config(config), + m_ccache_exe_dir(ccache_exe_dir) { } @@ -265,12 +426,6 @@ Storage::Storage(const Config& config) // RemoteStorageEntry and its constituents in the header file. Storage::~Storage() = default; -void -Storage::initialize() -{ - add_remote_storages(); -} - void Storage::finalize() { @@ -322,23 +477,17 @@ Storage::remove(const Hash::Digest& key, const core::CacheEntryType type) remove_from_remote_storage(key); } -bool -Storage::has_remote_storage() const -{ - return !m_remote_storages.empty(); -} - std::string Storage::get_remote_storage_config_for_logging() const { - auto configs = parse_storage_configs(m_config.remote_storage()); + auto configs = + parse_storage_configs(m_config.remote_storage(), ForLogging::yes); for (auto& config : configs) { const auto url = url_from_string(config.url_str); if (url) { - const auto storage = get_storage(url->scheme()); - if (storage) { - config.url_str = get_redacted_url_str_for_logging(*url); - storage->redact_secrets(config.attributes); + config.url_str = get_redacted_url_str_for_logging(*url); + for (auto& entry : config.attributes) { + entry.raw_value = k_redacted_secret; // could be credentials } } // else: unexpanded URL is not a proper URL, not much we can do } @@ -346,13 +495,25 @@ Storage::get_remote_storage_config_for_logging() const } void -Storage::add_remote_storages() +Storage::init_remote_storage() { - const auto configs = parse_storage_configs(m_config.remote_storage()); + if (!m_remote_storages.empty()) { + return; + } + + auto helper_search_dirs = util::getenv_path_list("PATH"); + const auto& libexec_dirs = m_config.libexec_dirs(); + helper_search_dirs.push_back(m_ccache_exe_dir); + helper_search_dirs.insert( + helper_search_dirs.end(), libexec_dirs.begin(), libexec_dirs.end()); + + const auto configs = + parse_storage_configs(m_config.remote_storage(), ForLogging::no); for (const auto& config : configs) { ASSERT(!config.shards.empty()); const std::string scheme = config.shards.front().url.scheme(); - const auto storage = get_storage(scheme); + const auto storage = get_remote_storage( + scheme, config, m_config.temporary_dir(), helper_search_dirs); if (!storage) { throw core::Error(FMT("unknown remote storage scheme: {}", scheme)); } @@ -367,6 +528,8 @@ Storage::mark_backend_as_failed( const remote::RemoteStorage::Backend::Failure failure) { // The backend is expected to log details about the error. + LOG("Marking remote storage backend for {} as failed", + backend_entry.url.scheme()); backend_entry.failed = true; local.increment_statistic( failure == remote::RemoteStorage::Backend::Failure::timeout @@ -462,6 +625,8 @@ Storage::get_from_remote_storage(const Hash::Digest& key, const core::CacheEntryType type, const EntryReceiver& entry_receiver) { + init_remote_storage(); + for (const auto& entry : m_remote_storages) { auto backend = get_backend(*entry, key, "getting from", false); if (!backend) { @@ -510,6 +675,8 @@ Storage::put_in_remote_storage(const Hash::Digest& key, return; } + init_remote_storage(); + for (const auto& entry : m_remote_storages) { auto backend = get_backend(*entry, key, "putting in", true); if (!backend) { @@ -538,6 +705,8 @@ Storage::put_in_remote_storage(const Hash::Digest& key, void Storage::remove_from_remote_storage(const Hash::Digest& key) { + init_remote_storage(); + for (const auto& entry : m_remote_storages) { auto backend = get_backend(*entry, key, "removing from", true); if (!backend) { @@ -569,4 +738,17 @@ Storage::remove_from_remote_storage(const Hash::Digest& key) } } +void +Storage::stop_remote_storage_helpers() +{ + init_remote_storage(); + + for (const auto& entry : m_remote_storages) { + const auto& config = entry->config; + for (const auto& shard : config.shards) { + entry->storage->create_backend(shard.url, config.attributes)->stop(); + } + } +} + } // namespace storage diff --git a/src/ccache/storage/storage.hpp b/src/ccache/storage/storage.hpp index 5fa22edcc..c48987f80 100644 --- a/src/ccache/storage/storage.hpp +++ b/src/ccache/storage/storage.hpp @@ -28,12 +28,15 @@ #include #include +#include #include #include #include #include #include +class Config; + namespace storage { constexpr auto k_redacted_secret = "********"; @@ -48,10 +51,9 @@ std::string get_redacted_url_str_for_logging(const Url& url); class Storage { public: - Storage(const Config& config); + Storage(const Config& config, const std::filesystem::path& ccache_exe_dir); ~Storage(); - void initialize(); void finalize(); local::LocalStorage local; @@ -68,14 +70,16 @@ public: void remove(const Hash::Digest& key, core::CacheEntryType type); - bool has_remote_storage() const; + void stop_remote_storage_helpers(); + std::string get_remote_storage_config_for_logging() const; private: const Config& m_config; + const std::filesystem::path& m_ccache_exe_dir; std::vector> m_remote_storages; - void add_remote_storages(); + void init_remote_storage(); void mark_backend_as_failed(RemoteStorageBackendEntry& backend_entry, remote::RemoteStorage::Backend::Failure failure); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ec1a09dc6..4bfc39ee7 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -58,6 +58,7 @@ addtest(profiling_hip_clang) addtest(readonly) addtest(readonly_direct) addtest(remote_file) +addtest(remote_helper) if(HTTP_STORAGE_BACKEND) addtest(remote_http) endif() diff --git a/test/run b/test/run index 8b53696c4..1a9c730ca 100755 --- a/test/run +++ b/test/run @@ -3,7 +3,7 @@ # A simple test suite for ccache. # # Copyright (C) 2002-2007 Andrew Tridgell -# Copyright (C) 2009-2025 Joel Rosdahl and other contributors +# Copyright (C) 2009-2026 Joel Rosdahl and other contributors # # See doc/authors.adoc for a complete list of contributors. # @@ -565,6 +565,8 @@ COMPILER_USES_MSVC=false ABS_ROOT_DIR="$(cd $(dirname "$0"); pwd)" readonly HTTP_CLIENT="${ABS_ROOT_DIR}/http-client" readonly HTTP_SERVER="${ABS_ROOT_DIR}/http-server" +readonly STORAGE_TEST_HELPER="$(pwd)/test/storage/helper/ccache-storage-test" +readonly STORAGE_TEST_CLIENT="$(pwd)/test/storage/client/storage-test-client" HOST_OS_APPLE=false HOST_OS_LINUX=false diff --git a/test/suites/remote_file.bash b/test/suites/remote_file.bash index 65bd84061..c170ae266 100644 --- a/test/suites/remote_file.bash +++ b/test/suites/remote_file.bash @@ -9,7 +9,7 @@ SUITE_remote_file_PROBE() { SUITE_remote_file_SETUP() { unset CCACHE_NODIRECT - export CCACHE_REMOTE_STORAGE="file:$PWD/remote" + export CCACHE_REMOTE_STORAGE="file:$PWD/remote helper=_builtin_" touch test.h echo '#include "test.h"' >test.c @@ -101,7 +101,7 @@ SUITE_remote_file() { # ------------------------------------------------------------------------- TEST "Flat layout" - CCACHE_REMOTE_STORAGE+="|layout=flat" + CCACHE_REMOTE_STORAGE+=" @layout=flat" $CCACHE_COMPILE -c test.c expect_stat direct_cache_hit 0 @@ -133,7 +133,7 @@ SUITE_remote_file() { # ------------------------------------------------------------------------- TEST "Two directories" - CCACHE_REMOTE_STORAGE+=" file://$PWD/remote_2" + CCACHE_REMOTE_STORAGE+=" file://$PWD/remote_2 helper=_builtin_" mkdir remote_2 $CCACHE_COMPILE -c test.c @@ -181,7 +181,7 @@ SUITE_remote_file() { expect_stat files_in_cache 0 expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest - CCACHE_REMOTE_STORAGE+="|read-only" + CCACHE_REMOTE_STORAGE+=" read-only" $CCACHE_COMPILE -c test.c expect_stat direct_cache_hit 1 @@ -272,7 +272,7 @@ SUITE_remote_file() { TEST "umask" export CCACHE_UMASK=042 - CCACHE_REMOTE_STORAGE="file://$PWD/remote|umask=024" + CCACHE_REMOTE_STORAGE="file://$PWD/remote helper=_builtin_ @umask=024" # local -> remote, cache miss $CCACHE_COMPILE -c test.c @@ -283,7 +283,7 @@ SUITE_remote_file() { expect_perm "${result_file}" -rw--w-r-- # 666 & 042 # local -> remote, local cache hit - CCACHE_REMOTE_STORAGE="file://$PWD/remote|umask=026" + CCACHE_REMOTE_STORAGE="file://$PWD/remote helper=_builtin_ @umask=026" $CCACHE -C >/dev/null rm -rf remote $CCACHE_COMPILE -c test.c @@ -305,7 +305,7 @@ SUITE_remote_file() { # ------------------------------------------------------------------------- TEST "Sharding" - CCACHE_REMOTE_STORAGE="file://$PWD/remote/*|shards=a,b(2)" + CCACHE_REMOTE_STORAGE="file://$PWD/remote/* helper=_builtin_ shards=a,b(2)" $CCACHE_COMPILE -c test.c expect_stat direct_cache_hit 0 @@ -318,7 +318,7 @@ SUITE_remote_file() { $CCACHE -Cz >/dev/null rm -rf remote - CCACHE_REMOTE_STORAGE="*|shards=file://$PWD/remote/a,file://$PWD/remote/b" + CCACHE_REMOTE_STORAGE="* helper=_builtin_ shards=file://$PWD/remote/a,file://$PWD/remote/b" $CCACHE_COMPILE -c test.c expect_stat direct_cache_hit 0 diff --git a/test/suites/remote_helper.bash b/test/suites/remote_helper.bash new file mode 100644 index 000000000..b77124c35 --- /dev/null +++ b/test/suites/remote_helper.bash @@ -0,0 +1,157 @@ +start_test_helper() { + local endpoint="$1" + + export CRSH_IPC_ENDPOINT="${endpoint}" + export CRSH_URL="dummy" + + "${STORAGE_TEST_HELPER}" & + + local attempts=0 + while [ $attempts -lt 50 ]; do + if "${STORAGE_TEST_CLIENT}" "${endpoint}" ping >/dev/null 2>&1; then + return 0 + fi + sleep 0.1 + attempts=$((attempts + 1)) + done + + test_failed_internal "Test storage helper failed to start (no response to ping)" +} + +SUITE_remote_helper_PROBE() { + if ! python3 --version >/dev/null 2>&1; then + echo "python3 is not available" + fi +} + +SUITE_remote_helper_SETUP() { + unset CCACHE_NODIRECT + + export CRSH_IDLE_TIMEOUT="10" + export CRSH_LOGFILE="ccache-storage-test.log" + generate_code 1 test.c +} + +SUITE_remote_helper() { + # ------------------------------------------------------------------------- + TEST "Helper auto-spawn and basic operations" + + export CCACHE_REMOTE_STORAGE="test://dummy helper=${STORAGE_TEST_HELPER}" + + # First compilation: miss, ccache spawns helper and stores + $CCACHE_COMPILE -c test.c + expect_stat direct_cache_hit 0 + expect_stat cache_miss 1 + expect_stat files_in_cache 2 + expect_stat remote_storage_hit 0 + expect_stat remote_storage_miss 1 + expect_stat remote_storage_read_hit 0 + expect_stat remote_storage_read_miss 2 # result + manifest + expect_stat remote_storage_write 2 # result + manifest + + # Second compilation: local hit (helper stays alive) + $CCACHE_COMPILE -c test.c + expect_stat direct_cache_hit 1 + expect_stat cache_miss 1 + expect_stat files_in_cache 2 + expect_stat remote_storage_hit 0 + expect_stat remote_storage_miss 1 + + # Clear local cache + $CCACHE -C >/dev/null + expect_stat files_in_cache 0 + + # Third compilation: remote hit from spawned helper + $CCACHE_COMPILE -c test.c + expect_stat direct_cache_hit 2 + expect_stat cache_miss 1 + expect_stat files_in_cache 2 # fetched from helper + expect_stat remote_storage_hit 1 + expect_stat remote_storage_miss 1 + expect_stat remote_storage_read_hit 2 # result + manifest + expect_stat remote_storage_read_miss 2 + expect_stat remote_storage_write 2 + + $CCACHE --stop-storage-helpers + + # ------------------------------------------------------------------------- + TEST "Helper reuse across compilations" + + export CCACHE_REMOTE_STORAGE="test://dummy2 helper=${STORAGE_TEST_HELPER}" + + # Multiple compilations should reuse same spawned helper + for i in 1 2 3; do + generate_code $i test.c + $CCACHE_COMPILE -c test.c + expect_stat cache_miss $i + done + + expect_stat files_in_cache 6 # 3 results + 3 manifests + + $CCACHE --stop-storage-helpers + + # ------------------------------------------------------------------------- + TEST "Direct crsh: connection" + + local endpoint="$PWD/test.sock" + + if $HOST_OS_WINDOWS; then + endpoint="ccache-test-$$" + fi + + start_test_helper "${endpoint}" + + export CCACHE_REMOTE_STORAGE="crsh:${endpoint}" + + # First compilation - miss, store to test helper + $CCACHE_COMPILE -c test.c + expect_stat direct_cache_hit 0 + expect_stat cache_miss 1 + expect_stat files_in_cache 2 + expect_stat remote_storage_hit 0 + expect_stat remote_storage_miss 1 + expect_stat remote_storage_read_hit 0 + expect_stat remote_storage_read_miss 2 # result + manifest + expect_stat remote_storage_write 2 # result + manifest + + # Second compilation - local hit + $CCACHE_COMPILE -c test.c + expect_stat direct_cache_hit 1 + expect_stat cache_miss 1 + expect_stat files_in_cache 2 + expect_stat remote_storage_hit 0 + expect_stat remote_storage_miss 1 + + # Clear local cache + $CCACHE -C >/dev/null + expect_stat files_in_cache 0 + + # Third compilation - remote hit from test helper + $CCACHE_COMPILE -c test.c + expect_stat direct_cache_hit 2 + expect_stat cache_miss 1 + expect_stat files_in_cache 2 # fetched from test helper + expect_stat remote_storage_hit 1 + expect_stat remote_storage_miss 1 + expect_stat remote_storage_read_hit 2 # result + manifest + expect_stat remote_storage_read_miss 2 + expect_stat remote_storage_write 2 + + # ------------------------------------------------------------------------- + TEST "Connection failure handling" + + local endpoint="$PWD/nonexistent.sock" + + if $HOST_OS_WINDOWS; then + endpoint="ccache-nonexistent-$$" + fi + + # Don't start helper - test connection failure + export CCACHE_REMOTE_STORAGE="crsh:${endpoint}" + + # Should fall back to local-only operation + $CCACHE_COMPILE -c test.c + expect_stat cache_miss 1 + expect_stat files_in_cache 2 + expect_stat remote_storage_error 1 +} diff --git a/test/suites/remote_http.bash b/test/suites/remote_http.bash index 33fc49ecc..072ceb16d 100644 --- a/test/suites/remote_http.bash +++ b/test/suites/remote_http.bash @@ -43,7 +43,7 @@ SUITE_remote_http() { TEST "Subdirs layout" start_http_server 12780 remote - export CCACHE_REMOTE_STORAGE="http://localhost:12780" + export CCACHE_REMOTE_STORAGE="http://localhost:12780 helper=_builtin_" $CCACHE_COMPILE -c test.c expect_stat direct_cache_hit 0 @@ -75,7 +75,7 @@ SUITE_remote_http() { TEST "Flat layout" start_http_server 12780 remote - export CCACHE_REMOTE_STORAGE="http://localhost:12780|layout=flat" + export CCACHE_REMOTE_STORAGE="http://localhost:12780 helper=_builtin_ @layout=flat" $CCACHE_COMPILE -c test.c expect_stat direct_cache_hit 0 @@ -108,7 +108,7 @@ SUITE_remote_http() { start_http_server 12780 remote mkdir remote/ac - export CCACHE_REMOTE_STORAGE="http://localhost:12780|layout=bazel" + export CCACHE_REMOTE_STORAGE="http://localhost:12780 helper=_builtin_ @layout=bazel" $CCACHE_COMPILE -c test.c expect_stat direct_cache_hit 0 @@ -139,7 +139,7 @@ SUITE_remote_http() { TEST "Basic auth" start_http_server 12780 remote "somebody:secret123" - export CCACHE_REMOTE_STORAGE="http://somebody:secret123@localhost:12780" + export CCACHE_REMOTE_STORAGE="http://somebody:secret123@localhost:12780 helper=_builtin_" CCACHE_DEBUG=1 $CCACHE_COMPILE -c test.c expect_stat direct_cache_hit 0 @@ -158,7 +158,7 @@ if $RUN_WIN_XFAIL; then start_http_server 12780 remote "somebody:secret123" # no authentication configured on client - export CCACHE_REMOTE_STORAGE="http://localhost:12780" + export CCACHE_REMOTE_STORAGE="http://localhost:12780 helper=_builtin_" CCACHE_DEBUG=1 $CCACHE_COMPILE -c test.c expect_stat direct_cache_hit 0 @@ -177,7 +177,7 @@ if $RUN_WIN_XFAIL; then TEST "Basic auth failed" start_http_server 12780 remote "somebody:secret123" - export CCACHE_REMOTE_STORAGE="http://somebody:wrong@localhost:12780" + export CCACHE_REMOTE_STORAGE="http://somebody:wrong@localhost:12780 helper=_builtin_" CCACHE_DEBUG=1 $CCACHE_COMPILE -c test.c expect_stat direct_cache_hit 0 @@ -192,7 +192,7 @@ fi TEST "Port sharding" start_http_server 12780 remote - export CCACHE_REMOTE_STORAGE="http://localhost:*|shards=12780" + export CCACHE_REMOTE_STORAGE="http://localhost:*|shards=12780 helper=_builtin_" $CCACHE_COMPILE -c test.c expect_stat direct_cache_hit 0 @@ -214,7 +214,7 @@ fi TEST "IPv6 address" if maybe_start_ipv6_http_server 12780 remote; then - export CCACHE_REMOTE_STORAGE="http://[::1]:12780" + export CCACHE_REMOTE_STORAGE="http://[::1]:12780 helper=_builtin_" $CCACHE_COMPILE -c test.c expect_stat direct_cache_hit 0 diff --git a/test/suites/remote_url.bash b/test/suites/remote_url.bash index d2c56ed8a..6a17c8fa8 100644 --- a/test/suites/remote_url.bash +++ b/test/suites/remote_url.bash @@ -3,20 +3,6 @@ SUITE_remote_url_SETUP() { } SUITE_remote_url() { - # ------------------------------------------------------------------------- - TEST "Reject empty url (without config attributes)" - - export CCACHE_REMOTE_STORAGE="|" - $CCACHE_COMPILE -c test.c 2>stderr.log - expect_contains stderr.log "must provide a URL" - - # ------------------------------------------------------------------------- - TEST "Reject empty url (but with config attributes)" - - export CCACHE_REMOTE_STORAGE="|key=value" - $CCACHE_COMPILE -c test.c 2>stderr.log - expect_contains stderr.log "must provide a URL" - # ------------------------------------------------------------------------- TEST "Reject invalid url" diff --git a/unittest/test_config.cpp b/unittest/test_config.cpp index 6624827a8..14a64a016 100644 --- a/unittest/test_config.cpp +++ b/unittest/test_config.cpp @@ -616,6 +616,7 @@ TEST_CASE("Config::visit_items") "ignore_options = -a=* -b\n" "inode_cache = false\n" "keep_comments_cpp = true\n" + "libexec_dirs = led\n" "log_file = lf\n" "max_files = 4711\n" "max_size = 98.7M\n" @@ -680,6 +681,7 @@ TEST_CASE("Config::visit_items") "(test.conf) ignore_options = -a=* -b", "(test.conf) inode_cache = false", "(test.conf) keep_comments_cpp = true", + "(test.conf) libexec_dirs = led", "(test.conf) log_file = lf", "(test.conf) max_files = 4711", "(test.conf) max_size = 98.7 MB", -- 2.47.3