cp "${arch}-binary/ccache" "${name}"
chmod +x "${name}/ccache"
cp misc/Makefile.posix-binary-release "${name}/Makefile"
+ cp misc/patch-binary.py "${name}"
cp GPL-3.0.txt README.md "${name}"
cp docs/install/usr/local/share/doc/ccache/* "${name}"
cp docs/install/usr/local/share/man/man1/ccache.1 "${name}"
set(INODE_CACHE_SUPPORTED 1)
endif()
-# Escape backslashes in SYSCONFDIR for C.
+file(TO_NATIVE_PATH "${CMAKE_INSTALL_FULL_LIBEXECDIR}" CONFIG_LIBEXECDIR_C_ESCAPED)
+string(REPLACE "\\" "\\\\" CONFIG_LIBEXECDIR_C_ESCAPED "${CONFIG_LIBEXECDIR_C_ESCAPED}")
file(TO_NATIVE_PATH "${CMAKE_INSTALL_FULL_SYSCONFDIR}" CONFIG_SYSCONFDIR_C_ESCAPED)
string(REPLACE "\\" "\\\\" CONFIG_SYSCONFDIR_C_ESCAPED "${CONFIG_SYSCONFDIR_C_ESCAPED}")
# define ESTALE -1
#endif
+#define LIBEXECDIR "@CONFIG_LIBEXECDIR_C_ESCAPED@"
#define SYSCONFDIR "@CONFIG_SYSCONFDIR_C_ESCAPED@"
#cmakedefine INODE_CACHE_SUPPORTED
format. Use `-v`/`--verbose` once or twice for more details. See
<<_cache_statistics,Cache statistics>> for more information.
+*--stop-storage-helpers*::
+
+ Storage helpers normally stop after a period of client inactivity, but this
+ option can be used to stop any running helpers immediately.
+
*--threads* _THREADS_::
Use up to _THREADS_ threads for threaded operations. The default is to use
output. The default is false. This can be used to check documentation with
`-Wdocumentation`.
+[#config_libexec_dir]
+*libexec_dirs* (*CCACHE_LIBEXEC_DIRS*)::
+
+ If set, ccache will look for helper programs in these directories instead of
+ the default. The list separator is semicolon on Windows systems and colon on
+ other systems.
+
[#config_log_file]
*log_file* (*CCACHE_LOGFILE*)::
----
remote_storage = file:/shared/nfs/directory+`
remote_storage =
- file:///shared/nfs/one|read-only
+ file:///shared/nfs/one read-only
file:///shared/nfs/two
remote_storage = file:///Z:/example/windows/folder
remote_storage = http://example.com/cache
kept in the local cache directory -- remote storage backends only store
compilation results and manifests.
-A remote storage backend is specified with a URL, optionally followed by a pipe
-(`|`) and a pipe-separated list of attributes. An attribute is _key_=_value_ or
-just _key_ as a short form of _key_=*true*. Attribute values must be
-https://en.wikipedia.org/wiki/Percent-encoding[percent-encoded] if they contain
-percent, pipe or space characters.
+=== Storage helper process
+
+ccache can spawn a long-lived local storage helper process (installed
+separately) to handle communication with the remote storage. Storage helpers
+keep remote connections up for efficiency and terminate after a while on
+inactivity.
+
+The storage helper program is called `ccache-storage-<scheme>`, where `<scheme>`
+is the scheme part of a URL, e.g. `https`. The program can be placed in these
+locations (in priority order):
-=== Common attributes
+1. In a location specified by the *helper* property (see below).
+2. In `$PATH`.
+3. Next to the ccache executable.
+4. In ccache's <<config_libexec_dirs,libexec directory>>.
-These optional attributes are available for all remote storage backends:
+If the program is not found, ccache falls back to one of the builtin backends if
+available, otherwise ccache exits with an error.
+=== Configuration syntax
+
+A remote storage backend is specified by a URL and can be followed by a
+whitespace-separated list of properties (`_key_=_value_`) and custom attributes
+(`@_key_=_value_`). A missing `=_value_` is shorthand for setting the value to
+*true*. Values must be
+https://en.wikipedia.org/wiki/Percent-encoding[percent-encoded] if they contain
+`%`, `|` or whitespace characters. For compatibility with older ccache versions,
+`|` characters are treated as spaces.
+
+==== Properties
+
+* *data-timeout*: Timeout for send/receive data transfer. Resets whenever data
+ is received or can be sent. Default: 1s.
+* *helper*: Override which storage helper to spawn (either a filename or a full
+ path). This is mainly useful when developing or testing new helpers.
+* *idle-timeout*: Timeout for the helper process to wait before exiting after
+ client inactivity (0s: stay up indefinitely). Default: 10m.
* *read-only*: If *true*, only read from this backend, don't write. The default
is *false*.
+* *request-timeout*: Timeout for the whole request. Default: 10s.
* *shards*: A comma-separated list of names for sharding (partitioning) the
cache entries using
https://en.wikipedia.org/wiki/Rendezvous_hashing[Rendezvous hashing],
to represent the available memory for a memory cache on a specific server. The
default weight is *1*.
+
+Timeouts can be specified with an ms (milliseconds), s (seconds), m (minutes), h
+(hours) or d (days) suffix.
++
Examples:
+
--
-* `+redis://cache-*.example.com|shards=a(3),b(1),c(1.5)+` will put 55% (3/5.5)
- of the cache on `+redis://cache-a.example.com+`, 18% (1/5.5) on
- `+redis://cache-b.example.com+` and 27% (1.5/5.5) on
- `+redis://cache-c.example.com+`.
-* `+http://example.com/*|shards=alpha,beta+` will put 50% of the cache on
+* `+http://cache-*.example.com shards=a(3),b(1),c(1.5)+` will put 55% (3/5.5)
+ of the cache on `+http://cache-a.example.com+`, 18% (1/5.5) on
+ `+http://cache-b.example.com+` and 27% (1.5/5.5) on
+ `+http://cache-c.example.com+`.
+* `+http://example.com/* shards=alpha,beta+` will put 50% of the cache on
`+http://example.com/alpha+` and 50% on `+http://example.com/beta+`.
--
+==== Custom attributes
+
+Custom attributes on the form `_@key_=_value_` are specific to each storage
+helper implementation. See the backend's documentation for which attributes are
+available. Examples:
+
+* `+file:///example/directory @umask=002 @update-mtime=true+`
+* `+https://example.com/path @bearer-token=deadbeef+`
=== Storage interaction
|==============================================================================
-^[1]^ Unless remote storage has attribute `read-only=true`. +
+^[1]^ Unless remote storage has property `read-only=true`. +
^[2]^ Unless local storage is set to share its cache hits with the
<<config_reshare,*reshare*>> option.
Examples:
* `+file:/shared/nfs/directory+`
-* `+file:///shared/nfs/directory|umask=002|update-mtime=true+`
+* `+file:///shared/nfs/directory @umask=002 @update-mtime=true+`
* `+file:///Z:/example/windows/folder+`
* `+file://example.com/shared/ccache%20folder+`
Optional attributes:
-* *layout*: How to store file under the cache directory. Available values:
+* *@layout*: How to store file under the cache directory. Available values:
+
--
* *flat*: Store all files directly under the cache directory.
--
+
The default is *subdirs*.
-* *umask*: This attribute (an octal integer) overrides the umask to use for
+* *@umask*: This attribute (an octal integer) overrides the umask to use for
files and directories in the cache directory.
-* *update-mtime*: If *true*, update the modification time (mtime) of cache
+* *@update-mtime*: If *true*, update the modification time (mtime) of cache
entries that are read. The default is *false*.
+=== CRSH storage backend
+
+URL format: `+crsh:IPC_ENDPOINT+`
+
+This backend connects to an already running <<Storage helper process,ccache
+storage helper>> (CRSH) process listening on `IPC_ENDPOINT` (Unix socket path or
+Windows named pipe). The use case for this is mainly when developing or
+debugging a storage helper, or in special setups where the helper program is
+started and managed externally.
+
+Examples:
+
+* `+crsh:/path/to/your.sock+` (Unix socket)
+* `+crsh:name-of-your-pipe+` (Windows named pipe)
+
+
=== HTTP storage backend
+WARNING: Built-in support for the HTTP protocol will likely be removed in a
+future ccache release.
+
URL format: `+http://HOST[:PORT][/PATH]+`
This backend stores data in an HTTP-compatible server. The required HTTP methods
* `+http://localhost+`
* `+http://someusername:p4ssw0rd@example.com/cache/+`
-* `+http://localhost:8080|layout=bazel|connect-timeout=50+`
+* `+http://localhost:8080 @layout=bazel+`
Optional attributes:
-* *bearer-token*: Bearer token used to authorize the HTTP requests.
-* *connect-timeout*: Timeout (in ms) for network connection. The default is 100.
-* *keep-alive*: If *true*, keep the HTTP connection to the storage server open
+* *@bearer-token*: Bearer token used to authorize the HTTP requests.
+* *@keep-alive*: If *true*, keep the HTTP connection to the storage server open
to avoid reconnects. The default is *true*.
-* *layout*: How to map key names to the path part of the URL. Available values:
+* *@layout*: How to map key names to the path part of the URL. Available values:
+
--
* *bazel*: Store values in a format compatible with the Bazel HTTP caching
--
+
The default is *subdirs*.
-* *header*: Add the key=value pair to the HTTP headers of the request. For example:
- `+header=Content-Type=application/octet-stream+` adds
- "Content-Type: application/octet-stream" to the http headers of the request.
-* *operation-timeout*: Timeout (in ms) for HTTP requests. The default is 10000.
-
+* *@header*: Add the key=value pair to the HTTP headers of the request. For
+ example: `+@header=Content-Type=application/octet-stream+` adds
+ "Content-Type: application/octet-stream" to the HTTP headers of the request.
=== Redis storage backend
+WARNING: Built-in support for the Redis protocol will likely be removed in a
+future ccache release.
+
URL formats:
`+redis://[[USERNAME:]PASSWORD@]HOST[:PORT][/DBNUMBER]+` +
TIP: See https://ccache.dev/howto/redis-storage.html[How to set up Redis
storage] for hints on setting up a Redis server for use with ccache.
-TIP: You can set up a cluster of Redis servers using the `shards` attribute
+TIP: You can set up a cluster of Redis servers using the `shards` property
described in _<<Remote storage backends>>_.
Examples:
* `+redis://localhost+`
-* `+redis://p4ssw0rd@cache.example.com:6379/0|connect-timeout=50+`
+* `+redis://p4ssw0rd@cache.example.com:6379/0+`
* `+redis+unix:/run/redis.sock+`
* `+redis+unix:///run/redis.sock+`
* `+redis+unix://p4ssw0rd@localhost/run/redis.sock?db=0+`
-Optional attributes:
-
-* *connect-timeout*: Timeout (in ms) for network connection. The default is 100.
-* *operation-timeout*: Timeout (in ms) for Redis commands. The default is 10000.
-
-
== Cache size management
By default, ccache has a 5 GB limit on the total size of files in the cache and
bindir = $(exec_prefix)/bin
datarootdir = $(prefix)/share
docdir = $(datarootdir)/doc/ccache
+libexecdir = $(exec_prefix)/libexec
mandir = $(datarootdir)/man
man1dir = $(mandir)/man1
sysconfdir = $(prefix)/etc
-default_sysconfdir = /usr/local/etc
doc_files = \
GPL-3.0.txt \
LICENSE.html \
install:
mkdir -p "$(DESTDIR)$(bindir)"
- $(PYTHON) -c 'import sys; sysconfdir = b"$(sysconfdir)"; default_sysconfdir = b"$(default_sysconfdir)"; sys.stdout.buffer.write(sys.stdin.buffer.read().replace(default_sysconfdir + b"\x00" * (4096 - len(default_sysconfdir)), sysconfdir + b"\x00" * (4096 - len(sysconfdir))))' <ccache >"$(DESTDIR)$(bindir)/ccache"
+ $(PYTHON) patch-binary.py "$(libexecdir)" "$(sysconfdir)" <ccache >"$(DESTDIR)$(bindir)/ccache"
chmod +x "$(DESTDIR)$(bindir)/ccache"
mkdir -p "$(DESTDIR)$(docdir)"
--- /dev/null
+#!/usr/bin/env python3
+
+import sys
+
+libexecdir = sys.argv[1]
+sysconfdir = sys.argv[2]
+
+replacements = [
+ (b"/usr/local/libexec", libexecdir.encode()),
+ (b"/usr/local/etc", sysconfdir.encode()),
+]
+ccache_bin = sys.stdin.buffer.read()
+for repl_from, repl_to in replacements:
+ ccache_bin = ccache_bin.replace(
+ repl_from + b"\x00" * (4096 - len(repl_from)),
+ repl_to + b"\x00" * (4096 - len(repl_to)),
+ )
+sys.stdout.buffer.write(ccache_bin)
}
}
- ctx.storage.initialize();
-
find_compiler(ctx, &find_non_ccache_executable, masquerading_as_compiler);
// Guess compiler after logging the config value in order to be able to
}
{
- Context ctx;
+ fs::path argv0 =
+#ifdef _WIN32
+ util::add_exe_suffix(argv[0]);
+#else
+ argv[0];
+#endif
+ const auto ccache_exe_dir =
+ fs::canonical(argv0).value_or(argv0).parent_path();
+ Context ctx(ccache_exe_dir);
ctx.initialize(std::move(argv_parts.compiler_and_args),
argv_parts.config_settings);
SignalHandler signal_handler(ctx);
if (!ctx.config.remote_only()) {
ctx.storage.local.increment_statistic(Statistic::local_storage_miss);
}
- if (ctx.storage.has_remote_storage()) {
+ if (!ctx.config.remote_storage().empty()) {
ctx.storage.local.increment_statistic(Statistic::remote_storage_miss);
}
} else if ((counters.get(Statistic::direct_cache_hit) > 0
ctx.config.set_depend_mode(false);
}
- if (ctx.storage.has_remote_storage()) {
+ if (!ctx.config.remote_storage().empty()) {
if (ctx.config.file_clone()) {
LOG_RAW("Disabling file clone mode since remote storage is enabled");
ctx.config.set_file_clone(false);
// Make room for binary patching at install time. The extra pointer to a buffer
// is needed to prevent the compiler from assuming too much about the string,
// such as its actual length.
+const char k_libexecdir_array[4096 + 1] = LIBEXECDIR;
+const char* k_libexecdir = k_libexecdir_array;
const char k_sysconfdir_array[4096 + 1] = SYSCONFDIR;
const char* k_sysconfdir = k_sysconfdir_array;
ignore_options,
inode_cache,
keep_comments_cpp,
+ libexec_dirs,
log_file,
max_files,
max_size,
{"ignore_options", {ConfigItem::ignore_options} },
{"inode_cache", {ConfigItem::inode_cache} },
{"keep_comments_cpp", {ConfigItem::keep_comments_cpp} },
+ {"libexec_dirs", {ConfigItem::libexec_dirs} },
{"log_file", {ConfigItem::log_file} },
{"max_files", {ConfigItem::max_files} },
{"max_size", {ConfigItem::max_size} },
{"IGNOREHEADERS", "ignore_headers_in_manifest"},
{"IGNOREOPTIONS", "ignore_options" },
{"INODECACHE", "inode_cache" },
+ {"LIBEXEC_DIRS", "libexec_dirs" },
{"LOGFILE", "log_file" },
{"MAXFILES", "max_files" },
{"MAXSIZE", "max_size" },
// system config).
}
+const char*
+Config::libexec_dir()
+{
+ return k_libexecdir;
+}
+
+const char*
+Config::sysconf_dir()
+{
+ return k_sysconfdir;
+}
+
const fs::path&
Config::config_path() const
{
case ConfigItem::keep_comments_cpp:
return format_bool(m_keep_comments_cpp);
+ case ConfigItem::libexec_dirs:
+ return util::join_path_list(m_libexec_dirs);
+
case ConfigItem::log_file:
return m_log_file.string();
m_keep_comments_cpp = parse_bool(value, env_var_key, negate);
break;
+ case ConfigItem::libexec_dirs:
+ m_libexec_dirs = util::split_path_list(value);
+ break;
+
case ConfigItem::log_file:
m_log_file = value;
break;
void read(const std::vector<std::string>& cmdline_config_settings = {});
+ static const char* libexec_dir();
+ static const char* sysconf_dir();
+
bool absolute_paths_in_stderr() const;
util::Args::ResponseFileFormat response_file_format() const;
const std::vector<std::filesystem::path>& base_dirs() const;
const std::string& ignore_options() const;
bool inode_cache() const;
bool keep_comments_cpp() const;
+ const std::vector<std::filesystem::path>& libexec_dirs() const;
const std::filesystem::path& log_file() const;
uint64_t max_files() const;
uint64_t max_size() const;
#endif
bool m_keep_comments_cpp = false;
std::filesystem::path m_log_file;
+ std::vector<std::filesystem::path> m_libexec_dirs{libexec_dir()};
uint64_t m_max_files = 0;
uint64_t m_max_size = 5ULL * 1024 * 1024 * 1024;
std::string m_msvc_dep_prefix = "Note: including file:";
return m_keep_comments_cpp;
}
+inline const std::vector<std::filesystem::path>&
+Config::libexec_dirs() const
+{
+ return m_libexec_dirs;
+}
+
inline const std::filesystem::path&
Config::log_file() const
{
namespace fs = util::filesystem;
-Context::Context()
+Context::Context(const fs::path& ccache_exe_dir)
: actual_cwd(fs::current_path().value_or("")),
apparent_cwd(util::apparent_cwd(actual_cwd)),
- storage(config),
+ storage(config, ccache_exe_dir),
#ifdef INODE_CACHE_SUPPORTED
inode_cache(config),
#endif
class Context : util::NonCopyable
{
public:
- Context();
+ Context(const std::filesystem::path& ccache_exe_dir = {});
~Context();
// Read configuration, initialize logging, etc. Typically not called from unit
-// Copyright (C) 2021-2025 Joel Rosdahl and other contributors
+// Copyright (C) 2021-2026 Joel Rosdahl and other contributors
//
// See doc/authors.adoc for a complete list of contributors.
//
PATH
--inspect PATH print result/manifest file at PATH in
human-readable format
+ --stop-storage-helpers stop running storage helper(s), if any
--print-log-stats print statistics counter IDs and corresponding
values from the stats log in machine-parsable
format
PRINT_STATS,
PRINT_VERSION,
SHOW_LOG_STATS,
+ STOP_STORAGE_HELPERS,
THREADS,
TRIM_DIR,
TRIM_MAX_SIZE,
const char options_string[] = "cCd:k:hF:M:po:svVxX:z";
const option long_options[] = {
- {"checksum-file", REQUIRED, nullptr, CHECKSUM_FILE },
- {"cleanup", NO_ARGUMENT, nullptr, 'c' },
- {"clear", NO_ARGUMENT, nullptr, 'C' },
- {"config-path", REQUIRED, nullptr, CONFIG_PATH },
- {"dir", REQUIRED, nullptr, 'd' },
- {"directory", REQUIRED, nullptr, 'd' }, // compat
- {"dump-manifest", REQUIRED, nullptr, INSPECT }, // compat
- {"dump-result", REQUIRED, nullptr, INSPECT }, // compat
- {"evict-namespace", REQUIRED, nullptr, EVICT_NAMESPACE },
- {"evict-older-than", REQUIRED, nullptr, EVICT_OLDER_THAN},
- {"extract-result", REQUIRED, nullptr, EXTRACT_RESULT },
- {"format", REQUIRED, nullptr, FORMAT },
- {"get-config", REQUIRED, nullptr, 'k' },
- {"hash-file", REQUIRED, nullptr, HASH_FILE },
- {"help", NO_ARGUMENT, nullptr, 'h' },
- {"inspect", REQUIRED, nullptr, INSPECT },
- {"max-files", REQUIRED, nullptr, 'F' },
- {"max-size", REQUIRED, nullptr, 'M' },
- {"print-log-stats", NO_ARGUMENT, nullptr, PRINT_LOG_STATS },
- {"print-stats", NO_ARGUMENT, nullptr, PRINT_STATS },
- {"print-version", NO_ARGUMENT, nullptr, PRINT_VERSION },
- {"recompress", REQUIRED, nullptr, 'X' },
- {"recompress-threads", REQUIRED, nullptr, THREADS }, // compat
- {"set-config", REQUIRED, nullptr, 'o' },
- {"show-compression", NO_ARGUMENT, nullptr, 'x' },
- {"show-config", NO_ARGUMENT, nullptr, 'p' },
- {"show-log-stats", NO_ARGUMENT, nullptr, SHOW_LOG_STATS },
- {"show-stats", NO_ARGUMENT, nullptr, 's' },
- {"threads", REQUIRED, nullptr, THREADS },
- {"trim-dir", REQUIRED, nullptr, TRIM_DIR },
- {"trim-max-size", REQUIRED, nullptr, TRIM_MAX_SIZE },
- {"trim-method", REQUIRED, nullptr, TRIM_METHOD },
- {"trim-recompress", REQUIRED, nullptr, TRIM_RECOMPRESS },
- {"trim-recompress-threads", REQUIRED, nullptr, THREADS }, // compat
- {"verbose", NO_ARGUMENT, nullptr, 'v' },
- {"version", NO_ARGUMENT, nullptr, 'V' },
- {"zero-stats", NO_ARGUMENT, nullptr, 'z' },
- {nullptr, 0, nullptr, 0 }
+ {"checksum-file", REQUIRED, nullptr, CHECKSUM_FILE },
+ {"cleanup", NO_ARGUMENT, nullptr, 'c' },
+ {"clear", NO_ARGUMENT, nullptr, 'C' },
+ {"config-path", REQUIRED, nullptr, CONFIG_PATH },
+ {"dir", REQUIRED, nullptr, 'd' },
+ {"directory", REQUIRED, nullptr, 'd' }, // compat
+ {"dump-manifest", REQUIRED, nullptr, INSPECT }, // compat
+ {"dump-result", REQUIRED, nullptr, INSPECT }, // compat
+ {"evict-namespace", REQUIRED, nullptr, EVICT_NAMESPACE },
+ {"evict-older-than", REQUIRED, nullptr, EVICT_OLDER_THAN },
+ {"extract-result", REQUIRED, nullptr, EXTRACT_RESULT },
+ {"format", REQUIRED, nullptr, FORMAT },
+ {"get-config", REQUIRED, nullptr, 'k' },
+ {"hash-file", REQUIRED, nullptr, HASH_FILE },
+ {"help", NO_ARGUMENT, nullptr, 'h' },
+ {"inspect", REQUIRED, nullptr, INSPECT },
+ {"max-files", REQUIRED, nullptr, 'F' },
+ {"max-size", REQUIRED, nullptr, 'M' },
+ {"print-log-stats", NO_ARGUMENT, nullptr, PRINT_LOG_STATS },
+ {"print-stats", NO_ARGUMENT, nullptr, PRINT_STATS },
+ {"print-version", NO_ARGUMENT, nullptr, PRINT_VERSION },
+ {"recompress", REQUIRED, nullptr, 'X' },
+ {"recompress-threads", REQUIRED, nullptr, THREADS }, // compat
+ {"set-config", REQUIRED, nullptr, 'o' },
+ {"show-compression", NO_ARGUMENT, nullptr, 'x' },
+ {"show-config", NO_ARGUMENT, nullptr, 'p' },
+ {"show-log-stats", NO_ARGUMENT, nullptr, SHOW_LOG_STATS },
+ {"show-stats", NO_ARGUMENT, nullptr, 's' },
+ {"stop-storage-helpers", NO_ARGUMENT, nullptr, STOP_STORAGE_HELPERS},
+ {"threads", REQUIRED, nullptr, THREADS },
+ {"trim-dir", REQUIRED, nullptr, TRIM_DIR },
+ {"trim-max-size", REQUIRED, nullptr, TRIM_MAX_SIZE },
+ {"trim-method", REQUIRED, nullptr, TRIM_METHOD },
+ {"trim-recompress", REQUIRED, nullptr, TRIM_RECOMPRESS },
+ {"trim-recompress-threads", REQUIRED, nullptr, THREADS }, // compat
+ {"verbose", NO_ARGUMENT, nullptr, 'v' },
+ {"version", NO_ARGUMENT, nullptr, 'V' },
+ {"zero-stats", NO_ARGUMENT, nullptr, 'z' },
+ {nullptr, 0, nullptr, 0 }
};
int
break;
}
+ case STOP_STORAGE_HELPERS: {
+ storage::Storage storage(config, fs::path(argv[0]).parent_path());
+ storage.stop_remote_storage_helpers();
+ break;
+ }
+
case TRIM_DIR:
if (!trim_max_size) {
throw Error("please specify --trim-max-size when using --trim-dir");
sources
client.cpp
filestorage.cpp
+ helper.cpp
remotestorage.cpp
)
util::value_or_throw<core::Fatal>(util::parse_umask(attr.value));
} else if (attr.key == "update-mtime") {
m_update_mtime = attr.value == "true";
- } else if (!is_framework_attribute(attr.key)) {
+ } else {
LOG("Unknown attribute: {}", attr.key);
}
}
--- /dev/null
+// Copyright (C) 2025-2026 Joel Rosdahl and other contributors
+//
+// See doc/authors.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "helper.hpp"
+
+#include <ccache/config.hpp>
+#include <ccache/core/exceptions.hpp>
+#include <ccache/hash.hpp>
+#include <ccache/storage/remote/client.hpp>
+#include <ccache/util/defer.hpp>
+#include <ccache/util/direntry.hpp>
+#include <ccache/util/environment.hpp>
+#include <ccache/util/error.hpp>
+#include <ccache/util/format.hpp>
+#include <ccache/util/lockfile.hpp>
+#include <ccache/util/logging.hpp>
+#include <ccache/util/process.hpp>
+#include <ccache/util/string.hpp>
+#include <ccache/util/timer.hpp>
+#include <ccache/util/wincompat.hpp>
+
+#ifndef _WIN32
+# include <dirent.h>
+# include <fcntl.h>
+# include <spawn.h>
+# include <sys/stat.h>
+# include <unistd.h>
+#endif
+
+#include <thread>
+
+#ifndef environ
+DLLIMPORT extern char** environ;
+#endif
+
+using namespace std::chrono_literals;
+
+namespace fs = util::filesystem;
+
+namespace storage::remote {
+
+namespace {
+
+#ifdef _WIN32
+constexpr std::string_view k_named_pipe_prefix = "\\\\.\\pipe\\";
+#endif
+
+// Call a function that returns 0 on success and either an error code or -1 (and
+// sets errno) on failure.
+#define CHECK_LIB_CALL(function, ...) \
+ do { \
+ int _result = function(__VA_ARGS__); \
+ if (_result != 0) { \
+ return tl::unexpected(FMT(#function " failed: {}", \
+ strerror(_result == -1 ? errno : _result))); \
+ } \
+ } while (false)
+
+// Generate a user-specific, unique socket/pipe path based on URL and
+// attributes.
+std::string
+generate_endpoint_name(
+ const Url& url,
+ const std::vector<RemoteStorage::Backend::Attribute>& attributes)
+{
+ static const uint8_t delimiter[1] = {0};
+
+ Hash hash;
+#ifdef _WIN32
+ char username[256];
+ DWORD username_len = sizeof(username);
+ if (GetUserNameA(username, &username_len)) {
+ hash.hash(username);
+ }
+#else
+ hash.hash(static_cast<int64_t>(getuid()));
+#endif
+ hash.hash(delimiter);
+ hash.hash(url.str());
+ for (const auto& attr : attributes) {
+ hash.hash(delimiter);
+ hash.hash(attr.key);
+ hash.hash(delimiter);
+ hash.hash(attr.value);
+ }
+ return FMT("storage-{}-{}", url.scheme(), util::format_base16(hash.digest()));
+}
+
+#ifndef _WIN32
+// Choose a short and safe base directory for Unix socket.
+//
+// Rationale:
+// - Unix socket paths have a strict length limit (sun_path).
+// - The ccache configured temporary dir can become very long (e.g. in CI).
+// - We want a directory that is private to the user to avoid other users
+// squatting the socket name.
+std::optional<fs::path>
+get_helper_ipc_dir()
+{
+ // If XDG_RUNTIME_DIR is set, use the same location as
+ // Config::default_temporary_dir.
+ if (const auto dir = Config::get_xdg_runtime_tmp_dir(); !dir.empty()) {
+ return dir;
+ }
+
+ // Otherwise, create a per-user private directory under /tmp. We intentionally
+ // use /tmp instead of $TMPDIR to keep socket paths short.
+ fs::path dir = FMT("/tmp/ccache-tmp-{}", static_cast<uint64_t>(getuid()));
+ if (auto r = fs::create_directories(dir); !r) {
+ LOG("Failed to create helper IPC dir {}: {}", dir, r.error());
+ return std::nullopt;
+ }
+
+ // Ensure correct permissions regardless of umask.
+ if (chmod(dir.c_str(), 0700) != 0) {
+ LOG("Failed to chmod helper IPC dir {}: {}", dir, strerror(errno));
+ return std::nullopt;
+ }
+
+ util::DirEntry entry(dir);
+ if (!entry) {
+ LOG("Failed to stat helper IPC dir {}: {}",
+ dir,
+ strerror(entry.error_number()));
+ return std::nullopt;
+ }
+ if (!entry.is_directory() || entry.is_symlink()) {
+ LOG("Helper IPC dir {} is not a directory", dir);
+ return std::nullopt;
+ }
+ if ((entry.mode() & 0077) != 0) {
+ LOG("Helper IPC dir {} is not private (mode {:o})", dir, entry.mode());
+ return std::nullopt;
+ }
+
+ return dir;
+}
+#endif
+
+std::vector<std::string>
+build_helper_env(
+ const Url& url,
+ std::string_view ipc_endpoint,
+ std::chrono::milliseconds idle_timeout,
+ const std::vector<RemoteStorage::Backend::Attribute>& attributes)
+{
+ std::vector<std::string> env_vars;
+ env_vars.emplace_back(FMT("CRSH_IPC_ENDPOINT={}", ipc_endpoint));
+ env_vars.emplace_back(FMT("CRSH_URL={}", url.str()));
+ env_vars.emplace_back(
+ FMT("CRSH_IDLE_TIMEOUT={}", idle_timeout.count() / 1000));
+ env_vars.emplace_back(FMT("CRSH_NUM_ATTR={}", attributes.size()));
+
+ for (size_t i = 0; i < attributes.size(); ++i) {
+ env_vars.emplace_back(FMT("CRSH_ATTR_KEY_{}={}", i, attributes[i].key));
+ env_vars.emplace_back(FMT("CRSH_ATTR_VALUE_{}={}", i, attributes[i].value));
+ }
+
+ return env_vars;
+}
+
+bool
+is_ccache_crsh_var(std::string_view entry)
+{
+ auto [name, value] = util::split_once(entry, '=');
+ if (!value) {
+ return false;
+ }
+
+ return name == "CRSH_IPC_ENDPOINT" || name == "CRSH_URL"
+ || name == "CRSH_IDLE_TIMEOUT" || name == "CRSH_NUM_ATTR"
+ || util::starts_with(name, "CRSH_ATTR_KEY_")
+ || util::starts_with(name, "CRSH_ATTR_VALUE_");
+}
+
+#ifndef _WIN32
+std::vector<int>
+get_fds_to_close()
+{
+ std::vector<int> fds_to_close;
+
+# ifdef __linux__
+ // Enumerate open FDs via /proc/self/fd for efficiency. (Using opendir instead
+ // of std::filesystem to be able to filter out dir_fd.)
+ bool enumerated = false;
+ if (DIR* dir = opendir("/proc/self/fd"); dir) {
+ enumerated = true;
+ const int dir_fd = dirfd(dir);
+ struct dirent* entry;
+ while ((entry = readdir(dir)) != nullptr) {
+ if (entry->d_name[0] == '.') {
+ continue; // skip . and ..
+ }
+ char* endptr;
+ long fd = std::strtol(entry->d_name, &endptr, 10);
+ if (*endptr == '\0' && fd >= 3 && fd != dir_fd) {
+ fds_to_close.push_back(static_cast<int>(fd));
+ }
+ }
+ closedir(dir);
+ }
+
+ if (enumerated) {
+ return fds_to_close;
+ }
+# endif
+
+ // Fallback: check FDs up to a reasonable limit.
+ long max_fd = sysconf(_SC_OPEN_MAX);
+ if (max_fd < 0 || max_fd > 1024) {
+ max_fd = 1024; // cap to avoid thousands of fcntl syscalls
+ }
+ for (int fd = 3; fd < max_fd; ++fd) {
+ // We must verify that the FD exists because on some systems (e.g. macOS),
+ // posix_spawn will fail when trying to close a non-existent FD.
+ if (fcntl(fd, F_GETFD) != -1) {
+ fds_to_close.push_back(fd);
+ }
+ }
+
+ return fds_to_close;
+}
+#endif
+
+tl::expected<void, std::string>
+spawn_helper(const fs::path& helper_path,
+ std::string_view endpoint,
+ const Url& url,
+ std::chrono::milliseconds idle_timeout,
+ const std::vector<RemoteStorage::Backend::Attribute>& attributes)
+{
+ LOG("Spawning storage helper {} for {}", helper_path, endpoint);
+
+#ifdef _WIN32
+ // Don't pass \\.\pipe\ prefix on Windows.
+ DEBUG_ASSERT(util::starts_with(endpoint, k_named_pipe_prefix));
+ auto ipc_endpoint = endpoint.substr(k_named_pipe_prefix.length());
+#else
+ const auto& ipc_endpoint = endpoint;
+#endif
+
+ const auto env_vars =
+ build_helper_env(url, ipc_endpoint, idle_timeout, attributes);
+
+#ifdef _WIN32
+ std::string env_block;
+ for (char** env = environ; *env != nullptr; ++env) {
+ if (!is_ccache_crsh_var(*env)) {
+ env_block += *env;
+ env_block += '\0';
+ }
+ }
+ for (const auto& var : env_vars) {
+ env_block += var;
+ env_block += '\0';
+ }
+ env_block += '\0';
+
+ STARTUPINFO si;
+ PROCESS_INFORMATION pi;
+ ZeroMemory(&si, sizeof(si));
+ si.cb = sizeof(si);
+ ZeroMemory(&pi, sizeof(pi));
+
+ std::string application = helper_path.string();
+ // CreateProcess may write into lpCommandLine, so it should be a mutable
+ // buffer != application.
+ std::string cmdline = application;
+ if (!CreateProcess(application.c_str(),
+ cmdline.data(),
+ nullptr,
+ nullptr,
+ FALSE,
+ CREATE_NO_WINDOW | DETACHED_PROCESS,
+ env_block.data(),
+ nullptr,
+ &si,
+ &pi)) {
+ DWORD error = GetLastError();
+ return tl::unexpected(
+ FMT("{} ({})", util::win32_error_message(error), error));
+ }
+
+ CloseHandle(pi.hThread);
+ CloseHandle(pi.hProcess);
+#else
+ std::vector<std::string> env_strings;
+ for (char** env = environ; *env != nullptr; ++env) {
+ if (!is_ccache_crsh_var(*env)) {
+ env_strings.emplace_back(*env);
+ }
+ }
+ env_strings.insert(env_strings.end(), env_vars.begin(), env_vars.end());
+
+ std::vector<char*> env_ptrs;
+ env_ptrs.reserve(env_strings.size() + 1);
+ for (auto& str : env_strings) {
+ env_ptrs.push_back(const_cast<char*>(str.c_str()));
+ }
+ env_ptrs.push_back(nullptr);
+
+ posix_spawn_file_actions_t actions;
+ CHECK_LIB_CALL(posix_spawn_file_actions_init, &actions);
+ DEFER(posix_spawn_file_actions_destroy(&actions));
+
+ CHECK_LIB_CALL(
+ posix_spawn_file_actions_addopen, &actions, 0, "/dev/null", O_RDONLY, 0);
+ CHECK_LIB_CALL(
+ posix_spawn_file_actions_addopen, &actions, 1, "/dev/null", O_WRONLY, 0);
+ CHECK_LIB_CALL(
+ posix_spawn_file_actions_addopen, &actions, 2, "/dev/null", O_WRONLY, 0);
+
+ // We need to close all inherited FDs since keeping them open in the
+ // long-lived helper process can interfere with build systems, see for example
+ // <https://github.com/ninja-build/ninja/issues/2052>.
+ for (int fd : get_fds_to_close()) {
+ CHECK_LIB_CALL(posix_spawn_file_actions_addclose, &actions, fd);
+ }
+
+ posix_spawnattr_t attr;
+ CHECK_LIB_CALL(posix_spawnattr_init, &attr);
+ DEFER(posix_spawnattr_destroy(&attr));
+
+ // Create a new session to fully detach from the controlling terminal.
+# ifdef POSIX_SPAWN_SETSID
+ CHECK_LIB_CALL(posix_spawnattr_setflags, &attr, POSIX_SPAWN_SETSID);
+# else
+ // Fallback for systems without POSIX_SPAWN_SETSID.
+ CHECK_LIB_CALL(posix_spawnattr_setflags, &attr, POSIX_SPAWN_SETPGROUP);
+ CHECK_LIB_CALL(posix_spawnattr_setpgroup, &attr, 0);
+# endif
+
+ pid_t pid;
+ char* argv[] = {const_cast<char*>(helper_path.c_str()), nullptr};
+ int result = posix_spawnp(
+ &pid, helper_path.c_str(), &actions, &attr, argv, env_ptrs.data());
+
+ if (result != 0) {
+ return tl::unexpected(strerror(result));
+ }
+
+ LOG("Spawned helper process with PID {}", pid);
+#endif
+
+ return {};
+}
+
+// Backend implementation that communicates with a helper process.
+class HelperBackend : public RemoteStorage::Backend
+{
+public:
+ HelperBackend(const fs::path& helper_path,
+ const fs::path& temp_dir,
+ const Url& url,
+ const std::vector<Backend::Attribute>& attributes,
+ std::chrono::milliseconds data_timeout,
+ std::chrono::milliseconds request_timeout,
+ std::chrono::milliseconds idle_timeout);
+
+ tl::expected<std::optional<util::Bytes>, Failure>
+ get(const Hash::Digest& key) override;
+
+ tl::expected<bool, Failure> put(const Hash::Digest& key,
+ nonstd::span<const uint8_t> value,
+ Overwrite overwrite) override;
+
+ tl::expected<bool, Failure> remove(const Hash::Digest& key) override;
+
+ void stop() override;
+
+private:
+ fs::path m_helper_path;
+ std::string m_endpoint; // Unix socket on POSIX, pipe name on Windows
+ fs::path m_endpoint_lock_path; // path to lock for guarding spawn of helper
+ Url m_url;
+ std::vector<Backend::Attribute> m_attributes;
+ std::chrono::milliseconds m_idle_timeout;
+ Client m_client;
+ bool m_connected = false;
+
+ tl::expected<void, Failure> ensure_connected(bool spawn = true);
+ tl::expected<void, Failure> finalize_connection();
+};
+
+HelperBackend::HelperBackend(const fs::path& helper_path,
+ const fs::path& temp_dir,
+ const Url& url,
+ const std::vector<Backend::Attribute>& attributes,
+ std::chrono::milliseconds data_timeout,
+ std::chrono::milliseconds request_timeout,
+ std::chrono::milliseconds idle_timeout)
+ : m_helper_path(helper_path),
+ m_url(url),
+ m_attributes(attributes),
+ m_idle_timeout(idle_timeout),
+ m_client(data_timeout, request_timeout)
+{
+ if (m_helper_path.empty()) {
+ // The "crsh:" URL case:
+#ifdef _WIN32
+ m_endpoint = FMT("{}{}", k_named_pipe_prefix, url.path());
+#else
+ m_endpoint = url.path();
+#endif
+ // No m_endpoint_lock_path needed since we won't spawn a helper.
+ } else {
+ // The common case:
+ auto endpoint_name = generate_endpoint_name(url, attributes);
+#ifdef _WIN32
+ m_endpoint = FMT("{}ccache-{}", k_named_pipe_prefix, endpoint_name);
+ m_endpoint_lock_path = FMT("{}/{}", temp_dir, endpoint_name);
+#else
+ auto helper_ipc_dir = get_helper_ipc_dir();
+ if (!helper_ipc_dir) {
+ LOG("Failed to select helper IPC dir, falling back to {}", temp_dir);
+ helper_ipc_dir = temp_dir;
+ }
+ m_endpoint = FMT("{}/{}", *helper_ipc_dir, endpoint_name);
+ m_endpoint_lock_path = m_endpoint;
+#endif
+ }
+}
+
+tl::expected<void, RemoteStorage::Backend::Failure>
+HelperBackend::finalize_connection()
+{
+ if (m_client.protocol_version() != Client::k_protocol_version) {
+ LOG("Unexpected remote storage helper protocol version: {} (!= {})",
+ m_client.protocol_version(),
+ Client::k_protocol_version);
+ return tl::unexpected(Failure::error);
+ }
+
+ if (!m_client.has_capability(Client::Capability::get_put_remove_stop)) {
+ LOG_RAW("Remote storage helper does not support capability 0");
+ return tl::unexpected(Failure::error);
+ }
+
+ m_connected = true;
+ return {};
+}
+
+tl::expected<void, RemoteStorage::Backend::Failure>
+HelperBackend::ensure_connected(bool spawn)
+{
+ if (m_connected) {
+ return {};
+ }
+
+ // Try to connect to an existing helper.
+ util::Timer timer;
+ auto connect_result = m_client.connect(m_endpoint);
+ if (connect_result) {
+ LOG("Connected to existing remote storage helper at {} ({:.2f} ms)",
+ m_endpoint,
+ timer.measure_ms());
+ return finalize_connection();
+ }
+ LOG(
+ "Failed to connect to existing remote storage helper at {}: {} ({:.2f} ms)",
+ m_endpoint,
+ connect_result.error().message,
+ timer.measure_ms());
+
+ if (!spawn) {
+ return {};
+ }
+
+ if (m_helper_path.empty()) {
+ // Could not connect to "crsh:" endpoint, so just fail.
+ return tl::unexpected(Failure::error);
+ }
+
+ // No existing helper, spawn a new one. Use a lock file to prevent multiple
+ // processes from spawning simultaneously.
+ util::LockFile spawn_lock(m_endpoint_lock_path);
+ if (!spawn_lock.acquire()) {
+ LOG_RAW("Failed to acquire spawn lock");
+ return tl::unexpected(Failure::error);
+ }
+
+ // We have the lock. Check again if another process spawned while we waited.
+ timer.reset();
+ if (m_client.connect(m_endpoint)) {
+ LOG(
+ "Connected to remote storage helper spawned by another process ({:.2f}"
+ " ms)",
+ timer.measure_ms());
+ return finalize_connection();
+ }
+
+ // No helper exists, spawn it now.
+ timer.reset();
+ auto spawn_result = spawn_helper(
+ m_helper_path, m_endpoint, m_url, m_idle_timeout, m_attributes);
+ if (!spawn_result) {
+ LOG("Failed to spawn helper: {}", spawn_result.error());
+ return tl::unexpected(Failure::error);
+ }
+ LOG("Spawned remote storage helper ({:.2f} ms)", timer.measure_ms());
+ timer.reset();
+
+ constexpr auto sleep_duration = 1ms;
+ constexpr double spawn_timeout_ms = 1000.0;
+
+ timer.reset();
+ while (timer.measure_ms() < spawn_timeout_ms) {
+ connect_result = m_client.connect(m_endpoint);
+ if (connect_result) {
+ LOG("Connected to newly spawned remote storage helper at {} ({:.2f} ms)",
+ m_endpoint,
+ timer.measure_ms());
+ return finalize_connection();
+ }
+
+ std::this_thread::sleep_for(sleep_duration);
+ }
+
+ LOG("Failed to connect to spawned remote storage helper: {}",
+ connect_result.error().message);
+
+ return tl::unexpected(Failure::timeout);
+}
+
+tl::expected<std::optional<util::Bytes>, RemoteStorage::Backend::Failure>
+HelperBackend::get(const Hash::Digest& key)
+{
+ TRY(ensure_connected());
+
+ auto result = m_client.get(key);
+ if (!result) {
+ const auto& error = result.error();
+ LOG("Remote storage get failed: {}", error.message);
+ auto failure = (error.failure == Client::Failure::timeout)
+ ? Failure::timeout
+ : Failure::error;
+ return tl::unexpected(failure);
+ }
+
+ return *result;
+}
+
+tl::expected<bool, RemoteStorage::Backend::Failure>
+HelperBackend::put(const Hash::Digest& key,
+ nonstd::span<const uint8_t> value,
+ Overwrite overwrite)
+{
+ TRY(ensure_connected());
+
+ Client::PutFlags flags;
+ flags.overwrite = (overwrite == Overwrite::yes);
+
+ auto result = m_client.put(key, value, flags);
+ if (!result) {
+ const auto& error = result.error();
+ LOG("Remote storage put failed: {}", error.message);
+ auto failure = (error.failure == Client::Failure::timeout)
+ ? Failure::timeout
+ : Failure::error;
+ return tl::unexpected(failure);
+ }
+
+ return *result;
+}
+
+tl::expected<bool, RemoteStorage::Backend::Failure>
+HelperBackend::remove(const Hash::Digest& key)
+{
+ TRY(ensure_connected());
+
+ auto result = m_client.remove(key);
+ if (!result) {
+ const auto& error = result.error();
+ LOG("Remote storage remove failed: {}", error.message);
+ auto failure = (error.failure == Client::Failure::timeout)
+ ? Failure::timeout
+ : Failure::error;
+ return tl::unexpected(failure);
+ }
+
+ return *result;
+}
+
+void
+HelperBackend::stop()
+{
+ if (auto r = ensure_connected(false); !r) {
+ LOG_RAW("Failed to connect to remote storage helper");
+ return;
+ }
+ if (!m_connected) {
+ LOG("No need to stop remote storage helper for {}", m_url.str());
+ return;
+ }
+ if (auto r = m_client.stop(); r) {
+ LOG("Stopped remote storage helper for {}", m_url.str());
+ } else {
+ LOG("Failed to stop remote storage helper for {}: {}",
+ m_url.str(),
+ r.error().message);
+ }
+}
+
+} // namespace
+
+Helper::Helper(const std::filesystem::path& helper_path,
+ const fs::path& temp_dir,
+ std::chrono::milliseconds data_timeout,
+ std::chrono::milliseconds request_timeout,
+ std::chrono::milliseconds idle_timeout)
+ : m_helper_path(helper_path),
+ m_temp_dir(temp_dir),
+ m_data_timeout(data_timeout),
+ m_request_timeout(request_timeout),
+ m_idle_timeout(idle_timeout)
+{
+}
+
+Helper::Helper(std::chrono::milliseconds data_timeout,
+ std::chrono::milliseconds request_timeout)
+ : m_data_timeout(data_timeout),
+ m_request_timeout(request_timeout)
+{
+}
+
+std::unique_ptr<RemoteStorage::Backend>
+Helper::create_backend(const Url& url,
+ const std::vector<Backend::Attribute>& attributes) const
+{
+ return std::make_unique<HelperBackend>(m_helper_path,
+ m_temp_dir,
+ url,
+ attributes,
+ m_data_timeout,
+ m_request_timeout,
+ m_idle_timeout);
+}
+
+} // namespace storage::remote
--- /dev/null
+// Copyright (C) 2025-2026 Joel Rosdahl and other contributors
+//
+// See doc/authors.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#pragma once
+
+#include <ccache/storage/remote/remotestorage.hpp>
+
+#include <chrono>
+#include <filesystem>
+#include <memory>
+#include <vector>
+
+namespace storage::remote {
+
+class Helper : public RemoteStorage
+{
+public:
+ Helper(const std::filesystem::path& helper_path,
+ const std::filesystem::path& temp_dir,
+ std::chrono::milliseconds data_timeout,
+ std::chrono::milliseconds request_timeout,
+ std::chrono::milliseconds idle_timeout);
+
+ // Special case: crsh scheme
+ Helper(std::chrono::milliseconds data_timeout,
+ std::chrono::milliseconds request_timeout);
+
+ std::unique_ptr<Backend> create_backend(
+ const Url& url,
+ const std::vector<Backend::Attribute>& attributes) const override;
+
+private:
+ std::filesystem::path m_helper_path; // empty -> connect to existing socket
+ std::filesystem::path m_temp_dir;
+ std::chrono::milliseconds m_data_timeout;
+ std::chrono::milliseconds m_request_timeout;
+ std::chrono::milliseconds m_idle_timeout;
+};
+
+} // namespace storage::remote
} else {
LOG("Incomplete header specification: {}", attr.value);
}
- } else if (!is_framework_attribute(attr.key)) {
+ } else {
LOG("Unknown attribute: {}", attr.key);
}
}
return std::make_unique<HttpStorageBackend>(url, attributes);
}
-void
-HttpStorage::redact_secrets(std::vector<Backend::Attribute>& attributes) const
-{
- auto bearer_token_attribute =
- std::find_if(attributes.begin(), attributes.end(), [&](const auto& attr) {
- return attr.key == "bearer-token";
- });
- if (bearer_token_attribute != attributes.end()) {
- bearer_token_attribute->value = storage::k_redacted_secret;
- bearer_token_attribute->raw_value = storage::k_redacted_secret;
- }
-}
-
} // namespace storage::remote
-// Copyright (C) 2021-2024 Joel Rosdahl and other contributors
+// Copyright (C) 2021-2026 Joel Rosdahl and other contributors
//
// See doc/authors.adoc for a complete list of contributors.
//
std::unique_ptr<Backend> create_backend(
const Url& url,
const std::vector<Backend::Attribute>& attributes) const override;
-
- void
- redact_secrets(std::vector<Backend::Attribute>& attributes) const override;
};
} // namespace storage::remote
connect_timeout = parse_timeout_attribute(attr.value);
} else if (attr.key == "operation-timeout") {
operation_timeout = parse_timeout_attribute(attr.value);
- } else if (!is_framework_attribute(attr.key)) {
+ } else {
LOG("Unknown attribute: {}", attr.key);
}
}
-// Copyright (C) 2021-2024 Joel Rosdahl and other contributors
+// Copyright (C) 2021-2026 Joel Rosdahl and other contributors
//
// See doc/authors.adoc for a complete list of contributors.
//
namespace storage::remote {
-bool
-RemoteStorage::Backend::is_framework_attribute(const std::string& name)
-{
- return name == "read-only" || name == "shards";
-}
-
std::chrono::milliseconds
RemoteStorage::Backend::parse_timeout_attribute(const std::string& value)
{
const auto k_default_connect_timeout = std::chrono::milliseconds{100};
const auto k_default_operation_timeout = std::chrono::milliseconds{10000};
+const auto k_default_data_timeout = std::chrono::seconds{1};
+const auto k_default_request_timeout = std::chrono::seconds{10};
+const auto k_default_idle_timeout = std::chrono::minutes{10};
+
// This class defines the API that a remote storage must implement.
class RemoteStorage
{
// removed, otherwise false.
virtual tl::expected<bool, Failure> remove(const Hash::Digest& key) = 0;
- // Determine whether an attribute is handled by the remote storage
- // framework itself.
- static bool is_framework_attribute(const std::string& name);
+ // Stop backend.
+ virtual void stop();
// Parse a timeout `value`, throwing `Failed` on error.
static std::chrono::milliseconds
virtual std::unique_ptr<Backend>
create_backend(const Url& url,
const std::vector<Backend::Attribute>& attributes) const = 0;
-
- // Redact secrets in backend attributes, if any.
- virtual void
- redact_secrets(std::vector<Backend::Attribute>& attributes) const;
};
// --- Inline implementations ---
-inline void
-RemoteStorage::redact_secrets(
- std::vector<Backend::Attribute>& /*attributes*/) const
-{
-}
-
inline RemoteStorage::Backend::Failed::Failed(Failure failure)
: Failed("", failure)
{
return m_failure;
}
+inline void
+RemoteStorage::Backend::stop()
+{
+}
+
} // namespace storage::remote
-// Copyright (C) 2021-2025 Joel Rosdahl and other contributors
+// Copyright (C) 2021-2026 Joel Rosdahl and other contributors
//
// See doc/authors.adoc for a complete list of contributors.
//
#include "storage.hpp"
#include <ccache/config.hpp>
+#include <ccache/context.hpp>
#include <ccache/core/cacheentry.hpp>
#include <ccache/core/exceptions.hpp>
#include <ccache/core/statistic.hpp>
#include <ccache/storage/remote/filestorage.hpp>
+#include <ccache/storage/remote/helper.hpp>
#ifdef HAVE_HTTP_STORAGE_BACKEND
# include <ccache/storage/remote/httpstorage.hpp>
#endif
#ifdef HAVE_REDIS_STORAGE_BACKEND
# include <ccache/storage/remote/redisstorage.hpp>
#endif
+#include <ccache/execute.hpp>
#include <ccache/util/assertions.hpp>
#include <ccache/util/bytes.hpp>
+#include <ccache/util/conversion.hpp>
+#include <ccache/util/environment.hpp>
#include <ccache/util/expected.hpp>
#include <ccache/util/file.hpp>
+#include <ccache/util/filesystem.hpp>
#include <ccache/util/format.hpp>
#include <ccache/util/logging.hpp>
#include <ccache/util/string.hpp>
#include <unordered_map>
#include <vector>
+using namespace std::chrono_literals;
+
+namespace fs = util::filesystem;
+
namespace storage {
-const std::unordered_map<std::string /*scheme*/,
+const std::unordered_map<std::string_view /*scheme*/,
std::shared_ptr<remote::RemoteStorage>>
- k_remote_storage_implementations = {
+ k_builtin_remote_storage_implementations = {
{"file", std::make_shared<remote::FileStorage>() },
#ifdef HAVE_HTTP_STORAGE_BACKEND
{"http", std::make_shared<remote::HttpStorage>() },
get_features()
{
std::vector<std::string> features;
- features.reserve(k_remote_storage_implementations.size());
- std::transform(k_remote_storage_implementations.begin(),
- k_remote_storage_implementations.end(),
+ features.reserve(k_builtin_remote_storage_implementations.size());
+ std::transform(k_builtin_remote_storage_implementations.begin(),
+ k_builtin_remote_storage_implementations.end(),
std::back_inserter(features),
[](auto& entry) { return FMT("{}-storage", entry.first); });
+ features.emplace_back("crsh-storage");
+ features.emplace_back("remote-storage");
return features;
}
// Representation of one entry in the remote_storage config option.
struct RemoteStorageConfig
{
- // Raw URL with unexpanded "*".
- std::string url_str;
-
- // "shard" attribute.
- std::vector<RemoteStorageShardConfig> shards;
+ std::string url_str; // Raw URL with unexpanded "*"
- // "read-only" attribute.
- bool read_only = false;
+ std::optional<std::chrono::milliseconds> data_timeout; // "data-timeout"
+ std::string helper; // "helper"
+ std::optional<std::chrono::milliseconds> idle_timeout; // "idle-timeout"
+ std::optional<bool> read_only; // "read-only"
+ std::optional<std::chrono::milliseconds> request_timeout; // "request-timeout"
+ std::vector<RemoteStorageShardConfig> shards; // "shards"
- // Other attributes.
+ // Custom attributes (@key=value).
std::vector<remote::RemoteStorage::Backend::Attribute> attributes;
};
};
static std::string
-to_string(const RemoteStorageConfig& entry)
+to_string(const storage::RemoteStorageConfig& entry)
{
std::string result = entry.url_str;
+
+ if (entry.data_timeout) {
+ result +=
+ FMT(" data-timeout={}", util::format_duration(*entry.data_timeout));
+ }
+ if (!entry.helper.empty()) {
+ result += FMT(" helper={}", entry.helper);
+ }
+ if (entry.idle_timeout) {
+ result +=
+ FMT(" idle-timeout={}", util::format_duration(*entry.idle_timeout));
+ }
+ if (entry.read_only) {
+ result += FMT(" read-only={}", *entry.read_only ? "true" : "false");
+ }
+ if (entry.request_timeout) {
+ result +=
+ FMT(" request-timeout={}", util::format_duration(*entry.request_timeout));
+ }
+ if (!entry.shards.empty() && !entry.shards[0].name.empty()) {
+ result += " shards=";
+ bool first = true;
+ for (auto shard : entry.shards) {
+ if (first) {
+ first = false;
+ } else {
+ result += ',';
+ }
+ result += FMT("{}({})", shard.name, shard.weight);
+ }
+ }
for (const auto& attr : entry.attributes) {
- result += FMT("|{}={}", attr.key, attr.raw_value);
+ result += FMT(" @{}={}", attr.key, attr.raw_value);
}
+
return result;
}
return url;
}
-static RemoteStorageConfig
-parse_storage_config(const std::string_view entry)
+static std::vector<RemoteStorageShardConfig>
+parse_shards(std::string_view url_str, std::string_view value)
{
- const auto parts =
- util::split_into_views(entry, "|", util::Tokenizer::Mode::include_empty);
+ std::vector<RemoteStorageShardConfig> result;
- if (parts.empty() || parts.front().empty()) {
+ const auto asterisk_count = std::count(url_str.begin(), url_str.end(), '*');
+ if (asterisk_count == 0) {
throw core::Error(
- FMT("remote storage config must provide a URL: {}", entry));
+ FMT(R"(Missing "*" in URL when using shards: "{}")", url_str));
+ } else if (asterisk_count > 1) {
+ throw core::Error(
+ FMT(R"(Multiple "*" in URL when using shards: "{}")", url_str));
}
+ std::string scheme;
+ for (const auto& shard : util::Tokenizer(value, ",")) {
+ double weight = 1.0;
+ std::string_view name;
+ const auto lp_pos = shard.find('(');
+ if (lp_pos != std::string_view::npos) {
+ if (shard.back() != ')') {
+ throw core::Error(FMT("Invalid shard name: \"{}\"", shard));
+ }
+ weight = util::value_or_throw<core::Error>(util::parse_double(
+ std::string(shard.substr(lp_pos + 1, shard.length() - lp_pos - 2))));
+ if (weight < 0.0) {
+ throw core::Error(FMT("Invalid shard weight: \"{}\"", weight));
+ }
+ name = shard.substr(0, lp_pos);
+ } else {
+ name = shard;
+ }
+ Url url = util::value_or_throw<core::Error>(
+ url_from_string(util::replace_first(url_str, "*", name)));
+ if (scheme.empty()) {
+ scheme = url.scheme();
+ } else if (url.scheme() != scheme) {
+ throw core::Error(
+ FMT("Expanding {} with shards results in different schemes ({} != {})",
+ url_str,
+ url.scheme(),
+ scheme));
+ }
+ result.push_back({std::string(name), weight, url});
+ }
+
+ return result;
+}
+
+namespace {
+
+enum class ForLogging { yes, no };
+
+}
+
+static RemoteStorageConfig
+parse_storage_config(const std::vector<std::string_view>::const_iterator& begin,
+ const std::vector<std::string_view>::const_iterator& end,
+ ForLogging for_logging)
+{
+ ASSERT(begin != end);
RemoteStorageConfig result;
- result.url_str = std::string(parts[0]);
- const auto& url_str = result.url_str;
- for (size_t i = 1; i < parts.size(); ++i) {
- if (parts[i].empty()) {
- continue;
- }
- const auto [key, right_hand_side] =
- util::split_once_into_views(parts[i], '=');
- const auto& raw_value = right_hand_side.value_or("true");
+ result.url_str = std::string(*begin);
+ auto& url_str = result.url_str;
+
+ for (auto part = begin + 1; part != end; ++part) {
+ auto parts = util::split_once_into_views(*part, '=');
+ auto key = parts.first;
+ auto right = parts.second;
+ const auto& raw_value = right.value_or("true");
const auto value =
util::value_or_throw<core::Error>(util::percent_decode(raw_value));
- if (key == "read-only") {
+
+ std::array backward_compat_attributes = {
+ "bearer-token",
+ "connect-timeout",
+ "header",
+ "keep-alive",
+ "layout",
+ "operation-timeout",
+ "umask",
+ "update-mtime",
+ };
+ std::string backward_compat_key;
+ if (std::any_of(backward_compat_attributes.begin(),
+ backward_compat_attributes.end(),
+ [&](const char* attr) { return key == attr; })) {
+ backward_compat_key = FMT("@{}", key);
+ key = backward_compat_key;
+ }
+
+ if (!key.empty() && key.front() == '@') {
+ result.attributes.push_back(
+ {std::string(key.substr(1)), value, std::string(raw_value)});
+ } else if (key == "data-timeout") {
+ result.data_timeout =
+ util::value_or_throw<core::Error>(util::parse_duration(value));
+ } else if (key == "idle-timeout") {
+ result.idle_timeout =
+ util::value_or_throw<core::Error>(util::parse_duration(value));
+ } else if (key == "helper") {
+ result.helper = value;
+ } else if (key == "read-only") {
result.read_only = (value == "true");
+ } else if (key == "request-timeout") {
+ result.request_timeout =
+ util::value_or_throw<core::Error>(util::parse_duration(value));
} else if (key == "shards") {
- const auto asterisk_count =
- std::count(url_str.begin(), url_str.end(), '*');
- if (asterisk_count == 0) {
- throw core::Error(
- FMT(R"(Missing "*" in URL when using shards: "{}")", url_str));
- } else if (asterisk_count > 1) {
- throw core::Error(
- FMT(R"(Multiple "*" in URL when using shards: "{}")", url_str));
- }
- std::string scheme;
- for (const auto& shard : util::Tokenizer(value, ",")) {
- double weight = 1.0;
- std::string_view name;
- const auto lp_pos = shard.find('(');
- if (lp_pos != std::string_view::npos) {
- if (shard.back() != ')') {
- throw core::Error(FMT("Invalid shard name: \"{}\"", shard));
- }
- weight =
- util::value_or_throw<core::Error>(util::parse_double(std::string(
- shard.substr(lp_pos + 1, shard.length() - lp_pos - 2))));
- if (weight < 0.0) {
- throw core::Error(FMT("Invalid shard weight: \"{}\"", weight));
- }
- name = shard.substr(0, lp_pos);
- } else {
- name = shard;
- }
-
- Url url = util::value_or_throw<core::Error>(
- url_from_string(util::replace_first(url_str, "*", name)));
- if (!scheme.empty() && url.scheme() != scheme) {
- throw core::Error(FMT("Scheme {} different from {} in {}",
- url.scheme(),
- scheme,
- url_str));
- }
- result.shards.push_back({std::string(name), weight, url});
- }
+ result.shards = parse_shards(url_str, value);
+ } else if (for_logging == ForLogging::no) {
+ LOG("Unknown remote config property: {}", *part);
}
-
- result.attributes.push_back(
- {std::string(key), value, std::string(raw_value)});
}
- // No shards => save the single URL as the sole shard.
if (result.shards.empty()) {
+ // No shards => save the single URL as the sole shard.
result.shards.push_back(
{"", 0.0, util::value_or_throw<core::Error>(url_from_string(url_str))});
}
}
static std::vector<RemoteStorageConfig>
-parse_storage_configs(const std::string_view& configs)
+parse_storage_configs(const std::string_view& config, ForLogging for_logging)
{
std::vector<RemoteStorageConfig> result;
- for (const auto& config : util::Tokenizer(configs, " ")) {
- result.push_back(parse_storage_config(config));
+
+ // Split on "|" as well for backward compatibility.
+ auto parts = util::split_into_views(config, " \t|");
+
+ auto it = parts.begin();
+ while (it != parts.end()) {
+ // Find end of config entry, i.e. next URL or end of parts.
+ auto it2 = it + 1;
+ while (it2 != parts.end()) {
+ auto [left, right] = util::split_once_into_views(*it2, ':');
+ if (right && left.find('=') == std::string_view::npos) {
+ // foo:... but not foo=...:... -> looks like a URL
+ break;
+ }
+ ++it2;
+ }
+ result.push_back(parse_storage_config(it, it2, for_logging));
+ it = it2;
}
+
return result;
}
+static fs::path
+find_remote_storage_helper(std::string_view name,
+ const std::vector<fs::path> search_dirs)
+{
+ bool bare_helper_name =
+#ifdef _WIN32
+ name.find('\\') == std::string::npos &&
+#endif
+ name.find('/') == std::string::npos;
+ fs::path path = bare_helper_name ? find_executable_in_path(name, search_dirs)
+#ifdef _WIN32
+ : util::add_exe_suffix(name);
+#else
+ : name;
+#endif
+
+ if (!path.empty() && fs::exists(path)) {
+ LOG("Found remote storage helper {}", path);
+ return path;
+ }
+
+ LOG("Could not find remote storage helper program \"{}\"", name);
+ return {};
+}
+
static std::shared_ptr<remote::RemoteStorage>
-get_storage(const std::string& scheme)
+get_remote_storage(std::string_view scheme,
+ const RemoteStorageConfig& config,
+ const fs::path& temp_dir,
+ const std::vector<fs::path>& helper_search_dirs)
{
- const auto it = k_remote_storage_implementations.find(scheme);
- if (it != k_remote_storage_implementations.end()) {
- return it->second;
+ // Special case: crsh scheme connects directly to storage helper socket.
+ if (scheme == "crsh") {
+ return std::make_shared<storage::remote::Helper>(
+ config.data_timeout.value_or(remote::k_default_data_timeout),
+ config.request_timeout.value_or(remote::k_default_request_timeout));
+ }
+
+ if (config.helper == "_builtin_" || scheme == "file") {
+ LOG_RAW("Forcing use of builtin storage backend");
} else {
- return {};
+ // Look up and use storage helper if available.
+ std::string helper_name =
+ !config.helper.empty() ? config.helper : FMT("ccache-storage-{}", scheme);
+ // TODO: If/when we in the future remove built-in http and redis backends,
+ // we can search for the helper program on demand instead.
+ fs::path helper_path =
+ find_remote_storage_helper(helper_name, helper_search_dirs);
+ if (!helper_path.empty()) {
+ return std::make_shared<storage::remote::Helper>(
+ helper_path,
+ temp_dir,
+ config.data_timeout.value_or(remote::k_default_data_timeout),
+ config.request_timeout.value_or(remote::k_default_request_timeout),
+ config.idle_timeout.value_or(remote::k_default_idle_timeout));
+ }
+ }
+
+ // Fall back to builtin implementation.
+ const auto it = k_builtin_remote_storage_implementations.find(scheme);
+ if (it != k_builtin_remote_storage_implementations.end()) {
+ return it->second;
}
+
+ return {};
}
std::string
return redacted_url.str();
}
-Storage::Storage(const Config& config)
+Storage::Storage(const Config& config, const fs::path& ccache_exe_dir)
: local(config),
- m_config(config)
+ m_config(config),
+ m_ccache_exe_dir(ccache_exe_dir)
{
}
// RemoteStorageEntry and its constituents in the header file.
Storage::~Storage() = default;
-void
-Storage::initialize()
-{
- add_remote_storages();
-}
-
void
Storage::finalize()
{
remove_from_remote_storage(key);
}
-bool
-Storage::has_remote_storage() const
-{
- return !m_remote_storages.empty();
-}
-
std::string
Storage::get_remote_storage_config_for_logging() const
{
- auto configs = parse_storage_configs(m_config.remote_storage());
+ auto configs =
+ parse_storage_configs(m_config.remote_storage(), ForLogging::yes);
for (auto& config : configs) {
const auto url = url_from_string(config.url_str);
if (url) {
- const auto storage = get_storage(url->scheme());
- if (storage) {
- config.url_str = get_redacted_url_str_for_logging(*url);
- storage->redact_secrets(config.attributes);
+ config.url_str = get_redacted_url_str_for_logging(*url);
+ for (auto& entry : config.attributes) {
+ entry.raw_value = k_redacted_secret; // could be credentials
}
} // else: unexpanded URL is not a proper URL, not much we can do
}
}
void
-Storage::add_remote_storages()
+Storage::init_remote_storage()
{
- const auto configs = parse_storage_configs(m_config.remote_storage());
+ if (!m_remote_storages.empty()) {
+ return;
+ }
+
+ auto helper_search_dirs = util::getenv_path_list("PATH");
+ const auto& libexec_dirs = m_config.libexec_dirs();
+ helper_search_dirs.push_back(m_ccache_exe_dir);
+ helper_search_dirs.insert(
+ helper_search_dirs.end(), libexec_dirs.begin(), libexec_dirs.end());
+
+ const auto configs =
+ parse_storage_configs(m_config.remote_storage(), ForLogging::no);
for (const auto& config : configs) {
ASSERT(!config.shards.empty());
const std::string scheme = config.shards.front().url.scheme();
- const auto storage = get_storage(scheme);
+ const auto storage = get_remote_storage(
+ scheme, config, m_config.temporary_dir(), helper_search_dirs);
if (!storage) {
throw core::Error(FMT("unknown remote storage scheme: {}", scheme));
}
const remote::RemoteStorage::Backend::Failure failure)
{
// The backend is expected to log details about the error.
+ LOG("Marking remote storage backend for {} as failed",
+ backend_entry.url.scheme());
backend_entry.failed = true;
local.increment_statistic(
failure == remote::RemoteStorage::Backend::Failure::timeout
const core::CacheEntryType type,
const EntryReceiver& entry_receiver)
{
+ init_remote_storage();
+
for (const auto& entry : m_remote_storages) {
auto backend = get_backend(*entry, key, "getting from", false);
if (!backend) {
return;
}
+ init_remote_storage();
+
for (const auto& entry : m_remote_storages) {
auto backend = get_backend(*entry, key, "putting in", true);
if (!backend) {
void
Storage::remove_from_remote_storage(const Hash::Digest& key)
{
+ init_remote_storage();
+
for (const auto& entry : m_remote_storages) {
auto backend = get_backend(*entry, key, "removing from", true);
if (!backend) {
}
}
+void
+Storage::stop_remote_storage_helpers()
+{
+ init_remote_storage();
+
+ for (const auto& entry : m_remote_storages) {
+ const auto& config = entry->config;
+ for (const auto& shard : config.shards) {
+ entry->storage->create_backend(shard.url, config.attributes)->stop();
+ }
+ }
+}
+
} // namespace storage
#include <nonstd/span.hpp>
#include <cstdint>
+#include <filesystem>
#include <functional>
#include <memory>
#include <string>
#include <string_view>
#include <vector>
+class Config;
+
namespace storage {
constexpr auto k_redacted_secret = "********";
class Storage
{
public:
- Storage(const Config& config);
+ Storage(const Config& config, const std::filesystem::path& ccache_exe_dir);
~Storage();
- void initialize();
void finalize();
local::LocalStorage local;
void remove(const Hash::Digest& key, core::CacheEntryType type);
- bool has_remote_storage() const;
+ void stop_remote_storage_helpers();
+
std::string get_remote_storage_config_for_logging() const;
private:
const Config& m_config;
+ const std::filesystem::path& m_ccache_exe_dir;
std::vector<std::unique_ptr<RemoteStorageEntry>> m_remote_storages;
- void add_remote_storages();
+ void init_remote_storage();
void mark_backend_as_failed(RemoteStorageBackendEntry& backend_entry,
remote::RemoteStorage::Backend::Failure failure);
addtest(readonly)
addtest(readonly_direct)
addtest(remote_file)
+addtest(remote_helper)
if(HTTP_STORAGE_BACKEND)
addtest(remote_http)
endif()
# A simple test suite for ccache.
#
# Copyright (C) 2002-2007 Andrew Tridgell
-# Copyright (C) 2009-2025 Joel Rosdahl and other contributors
+# Copyright (C) 2009-2026 Joel Rosdahl and other contributors
#
# See doc/authors.adoc for a complete list of contributors.
#
ABS_ROOT_DIR="$(cd $(dirname "$0"); pwd)"
readonly HTTP_CLIENT="${ABS_ROOT_DIR}/http-client"
readonly HTTP_SERVER="${ABS_ROOT_DIR}/http-server"
+readonly STORAGE_TEST_HELPER="$(pwd)/test/storage/helper/ccache-storage-test"
+readonly STORAGE_TEST_CLIENT="$(pwd)/test/storage/client/storage-test-client"
HOST_OS_APPLE=false
HOST_OS_LINUX=false
SUITE_remote_file_SETUP() {
unset CCACHE_NODIRECT
- export CCACHE_REMOTE_STORAGE="file:$PWD/remote"
+ export CCACHE_REMOTE_STORAGE="file:$PWD/remote helper=_builtin_"
touch test.h
echo '#include "test.h"' >test.c
# -------------------------------------------------------------------------
TEST "Flat layout"
- CCACHE_REMOTE_STORAGE+="|layout=flat"
+ CCACHE_REMOTE_STORAGE+=" @layout=flat"
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 0
# -------------------------------------------------------------------------
TEST "Two directories"
- CCACHE_REMOTE_STORAGE+=" file://$PWD/remote_2"
+ CCACHE_REMOTE_STORAGE+=" file://$PWD/remote_2 helper=_builtin_"
mkdir remote_2
$CCACHE_COMPILE -c test.c
expect_stat files_in_cache 0
expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
- CCACHE_REMOTE_STORAGE+="|read-only"
+ CCACHE_REMOTE_STORAGE+=" read-only"
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 1
TEST "umask"
export CCACHE_UMASK=042
- CCACHE_REMOTE_STORAGE="file://$PWD/remote|umask=024"
+ CCACHE_REMOTE_STORAGE="file://$PWD/remote helper=_builtin_ @umask=024"
# local -> remote, cache miss
$CCACHE_COMPILE -c test.c
expect_perm "${result_file}" -rw--w-r-- # 666 & 042
# local -> remote, local cache hit
- CCACHE_REMOTE_STORAGE="file://$PWD/remote|umask=026"
+ CCACHE_REMOTE_STORAGE="file://$PWD/remote helper=_builtin_ @umask=026"
$CCACHE -C >/dev/null
rm -rf remote
$CCACHE_COMPILE -c test.c
# -------------------------------------------------------------------------
TEST "Sharding"
- CCACHE_REMOTE_STORAGE="file://$PWD/remote/*|shards=a,b(2)"
+ CCACHE_REMOTE_STORAGE="file://$PWD/remote/* helper=_builtin_ shards=a,b(2)"
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 0
$CCACHE -Cz >/dev/null
rm -rf remote
- CCACHE_REMOTE_STORAGE="*|shards=file://$PWD/remote/a,file://$PWD/remote/b"
+ CCACHE_REMOTE_STORAGE="* helper=_builtin_ shards=file://$PWD/remote/a,file://$PWD/remote/b"
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 0
--- /dev/null
+start_test_helper() {
+ local endpoint="$1"
+
+ export CRSH_IPC_ENDPOINT="${endpoint}"
+ export CRSH_URL="dummy"
+
+ "${STORAGE_TEST_HELPER}" &
+
+ local attempts=0
+ while [ $attempts -lt 50 ]; do
+ if "${STORAGE_TEST_CLIENT}" "${endpoint}" ping >/dev/null 2>&1; then
+ return 0
+ fi
+ sleep 0.1
+ attempts=$((attempts + 1))
+ done
+
+ test_failed_internal "Test storage helper failed to start (no response to ping)"
+}
+
+SUITE_remote_helper_PROBE() {
+ if ! python3 --version >/dev/null 2>&1; then
+ echo "python3 is not available"
+ fi
+}
+
+SUITE_remote_helper_SETUP() {
+ unset CCACHE_NODIRECT
+
+ export CRSH_IDLE_TIMEOUT="10"
+ export CRSH_LOGFILE="ccache-storage-test.log"
+ generate_code 1 test.c
+}
+
+SUITE_remote_helper() {
+ # -------------------------------------------------------------------------
+ TEST "Helper auto-spawn and basic operations"
+
+ export CCACHE_REMOTE_STORAGE="test://dummy helper=${STORAGE_TEST_HELPER}"
+
+ # First compilation: miss, ccache spawns helper and stores
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 0
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2
+ expect_stat remote_storage_hit 0
+ expect_stat remote_storage_miss 1
+ expect_stat remote_storage_read_hit 0
+ expect_stat remote_storage_read_miss 2 # result + manifest
+ expect_stat remote_storage_write 2 # result + manifest
+
+ # Second compilation: local hit (helper stays alive)
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 1
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2
+ expect_stat remote_storage_hit 0
+ expect_stat remote_storage_miss 1
+
+ # Clear local cache
+ $CCACHE -C >/dev/null
+ expect_stat files_in_cache 0
+
+ # Third compilation: remote hit from spawned helper
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 2
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2 # fetched from helper
+ expect_stat remote_storage_hit 1
+ expect_stat remote_storage_miss 1
+ expect_stat remote_storage_read_hit 2 # result + manifest
+ expect_stat remote_storage_read_miss 2
+ expect_stat remote_storage_write 2
+
+ $CCACHE --stop-storage-helpers
+
+ # -------------------------------------------------------------------------
+ TEST "Helper reuse across compilations"
+
+ export CCACHE_REMOTE_STORAGE="test://dummy2 helper=${STORAGE_TEST_HELPER}"
+
+ # Multiple compilations should reuse same spawned helper
+ for i in 1 2 3; do
+ generate_code $i test.c
+ $CCACHE_COMPILE -c test.c
+ expect_stat cache_miss $i
+ done
+
+ expect_stat files_in_cache 6 # 3 results + 3 manifests
+
+ $CCACHE --stop-storage-helpers
+
+ # -------------------------------------------------------------------------
+ TEST "Direct crsh: connection"
+
+ local endpoint="$PWD/test.sock"
+
+ if $HOST_OS_WINDOWS; then
+ endpoint="ccache-test-$$"
+ fi
+
+ start_test_helper "${endpoint}"
+
+ export CCACHE_REMOTE_STORAGE="crsh:${endpoint}"
+
+ # First compilation - miss, store to test helper
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 0
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2
+ expect_stat remote_storage_hit 0
+ expect_stat remote_storage_miss 1
+ expect_stat remote_storage_read_hit 0
+ expect_stat remote_storage_read_miss 2 # result + manifest
+ expect_stat remote_storage_write 2 # result + manifest
+
+ # Second compilation - local hit
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 1
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2
+ expect_stat remote_storage_hit 0
+ expect_stat remote_storage_miss 1
+
+ # Clear local cache
+ $CCACHE -C >/dev/null
+ expect_stat files_in_cache 0
+
+ # Third compilation - remote hit from test helper
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 2
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2 # fetched from test helper
+ expect_stat remote_storage_hit 1
+ expect_stat remote_storage_miss 1
+ expect_stat remote_storage_read_hit 2 # result + manifest
+ expect_stat remote_storage_read_miss 2
+ expect_stat remote_storage_write 2
+
+ # -------------------------------------------------------------------------
+ TEST "Connection failure handling"
+
+ local endpoint="$PWD/nonexistent.sock"
+
+ if $HOST_OS_WINDOWS; then
+ endpoint="ccache-nonexistent-$$"
+ fi
+
+ # Don't start helper - test connection failure
+ export CCACHE_REMOTE_STORAGE="crsh:${endpoint}"
+
+ # Should fall back to local-only operation
+ $CCACHE_COMPILE -c test.c
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2
+ expect_stat remote_storage_error 1
+}
TEST "Subdirs layout"
start_http_server 12780 remote
- export CCACHE_REMOTE_STORAGE="http://localhost:12780"
+ export CCACHE_REMOTE_STORAGE="http://localhost:12780 helper=_builtin_"
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 0
TEST "Flat layout"
start_http_server 12780 remote
- export CCACHE_REMOTE_STORAGE="http://localhost:12780|layout=flat"
+ export CCACHE_REMOTE_STORAGE="http://localhost:12780 helper=_builtin_ @layout=flat"
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 0
start_http_server 12780 remote
mkdir remote/ac
- export CCACHE_REMOTE_STORAGE="http://localhost:12780|layout=bazel"
+ export CCACHE_REMOTE_STORAGE="http://localhost:12780 helper=_builtin_ @layout=bazel"
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 0
TEST "Basic auth"
start_http_server 12780 remote "somebody:secret123"
- export CCACHE_REMOTE_STORAGE="http://somebody:secret123@localhost:12780"
+ export CCACHE_REMOTE_STORAGE="http://somebody:secret123@localhost:12780 helper=_builtin_"
CCACHE_DEBUG=1 $CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 0
start_http_server 12780 remote "somebody:secret123"
# no authentication configured on client
- export CCACHE_REMOTE_STORAGE="http://localhost:12780"
+ export CCACHE_REMOTE_STORAGE="http://localhost:12780 helper=_builtin_"
CCACHE_DEBUG=1 $CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 0
TEST "Basic auth failed"
start_http_server 12780 remote "somebody:secret123"
- export CCACHE_REMOTE_STORAGE="http://somebody:wrong@localhost:12780"
+ export CCACHE_REMOTE_STORAGE="http://somebody:wrong@localhost:12780 helper=_builtin_"
CCACHE_DEBUG=1 $CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 0
TEST "Port sharding"
start_http_server 12780 remote
- export CCACHE_REMOTE_STORAGE="http://localhost:*|shards=12780"
+ export CCACHE_REMOTE_STORAGE="http://localhost:*|shards=12780 helper=_builtin_"
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 0
TEST "IPv6 address"
if maybe_start_ipv6_http_server 12780 remote; then
- export CCACHE_REMOTE_STORAGE="http://[::1]:12780"
+ export CCACHE_REMOTE_STORAGE="http://[::1]:12780 helper=_builtin_"
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 0
}
SUITE_remote_url() {
- # -------------------------------------------------------------------------
- TEST "Reject empty url (without config attributes)"
-
- export CCACHE_REMOTE_STORAGE="|"
- $CCACHE_COMPILE -c test.c 2>stderr.log
- expect_contains stderr.log "must provide a URL"
-
- # -------------------------------------------------------------------------
- TEST "Reject empty url (but with config attributes)"
-
- export CCACHE_REMOTE_STORAGE="|key=value"
- $CCACHE_COMPILE -c test.c 2>stderr.log
- expect_contains stderr.log "must provide a URL"
-
# -------------------------------------------------------------------------
TEST "Reject invalid url"
"ignore_options = -a=* -b\n"
"inode_cache = false\n"
"keep_comments_cpp = true\n"
+ "libexec_dirs = led\n"
"log_file = lf\n"
"max_files = 4711\n"
"max_size = 98.7M\n"
"(test.conf) ignore_options = -a=* -b",
"(test.conf) inode_cache = false",
"(test.conf) keep_comments_cpp = true",
+ "(test.conf) libexec_dirs = led",
"(test.conf) log_file = lf",
"(test.conf) max_files = 4711",
"(test.conf) max_size = 98.7 MB",
projects / thirdparty / ccache.git / commitdiff
