=== Redis storage backend
URL format: `+redis://[[USERNAME:]PASSWORD@]HOST[:PORT][/DBNUMBER]+`
+Alternative: `+redis+unix://[[USERNAME:]PASSWORD@localhost]SOCKET_PATH[?db=DBNUMBER]+`
This backend stores data in a https://redis.io[Redis] (or Redis-compatible)
server. There are implementations for both memory-based and disk-based storage.
* `+redis://localhost+`
* `+redis://p4ssw0rd@cache.example.com:6379/0|connect-timeout=50+`
+* `+redis+unix:/run/redis.sock+`
+* `+redis+unix:///run/redis.sock+`
+* `+redis+unix://p4ssw0rd@localhost/var/run/redis/redis-server.sock?db=0|connect-timeout=50+`
Optional attributes:
{"http", std::make_shared<secondary::HttpStorage>()},
#ifdef HAVE_REDIS_STORAGE_BACKEND
{"redis", std::make_shared<secondary::RedisStorage>()},
+ {"redis+unix", std::make_shared<secondary::RedisStorage>()},
#endif
};
return util::join(configs, " ");
}
+static void
+redact_url_for_logging(Url& url_for_logging)
+{
+ url_for_logging.user_info("");
+}
+
void
Storage::add_secondary_storages()
{
const auto configs = parse_storage_configs(m_config.secondary_storage());
for (const auto& config : configs) {
auto url_for_logging = config.params.url;
- url_for_logging.user_info("");
+ redact_url_for_logging(url_for_logging);
const auto storage = get_storage(config.params.url);
if (!storage) {
throw core::Error("unknown secondary storage URL: {}",
if (backend == entry.backends.end()) {
auto shard_url_for_logging = shard_url;
- shard_url_for_logging.user_info("");
+ redact_url_for_logging(shard_url_for_logging);
entry.backends.push_back(
{shard_url, shard_url_for_logging.str(), {}, false});
auto shard_params = entry.config.params;
#endif
#include <cstdarg>
+#include <map>
#include <memory>
namespace storage::secondary {
}
}
+std::map<std::string, std::string>
+split_parameters(const Url::Query& query)
+{
+ std::map<std::string, std::string> m;
+ if (!query.empty()) {
+ auto it = query.begin();
+ auto end = query.end();
+ do {
+ m[it->key()] = it->val();
+ } while (++it != end);
+ }
+ return m;
+}
+
RedisStorageBackend::RedisStorageBackend(const Params& params)
: m_prefix("ccache"), // TODO: attribute
m_context(nullptr, redisFree)
{
const auto& url = params.url;
- ASSERT(url.scheme() == "redis");
+ ASSERT(url.scheme() == "redis" || url.scheme() == "redis+unix");
+ if (url.scheme() == "redis+unix" && !params.url.host().empty()
+ && params.url.host() != "localhost") {
+ throw core::Fatal(FMT(
+ "invalid file path \"{}\": specifying a host (\"{}\") is not supported",
+ params.url.str(),
+ params.url.host()));
+ }
auto connect_timeout = k_default_connect_timeout;
auto operation_timeout = k_default_operation_timeout;
const uint32_t connect_timeout,
const uint32_t operation_timeout)
{
- const std::string host = url.host().empty() ? "localhost" : url.host();
- const uint32_t port = url.port().empty()
- ? DEFAULT_PORT
- : util::value_or_throw<core::Fatal>(
- util::parse_unsigned(url.port(), 1, 65535, "port"));
- ASSERT(url.path().empty() || url.path()[0] == '/');
-
- LOG("Redis connecting to {}:{} (connect timeout {} ms)",
- host,
- port,
- connect_timeout);
- m_context.reset(
- redisConnectWithTimeout(host.c_str(), port, to_timeval(connect_timeout)));
+ if (url.scheme() == "redis+unix") {
+ LOG("Redis connecting to unix://{} (connect timeout {} ms)",
+ url.path(),
+ connect_timeout);
+ m_context.reset(redisConnectUnixWithTimeout(url.path().c_str(),
+ to_timeval(connect_timeout)));
+ } else {
+ const std::string host = url.host().empty() ? "localhost" : url.host();
+ const uint32_t port =
+ url.port().empty() ? DEFAULT_PORT
+ : util::value_or_throw<core::Fatal>(
+ util::parse_unsigned(url.port(), 1, 65535, "port"));
+ ASSERT(url.path().empty() || url.path()[0] == '/');
+
+ LOG("Redis connecting to {}:{} (connect timeout {} ms)",
+ host,
+ port,
+ connect_timeout);
+ m_context.reset(
+ redisConnectWithTimeout(host.c_str(), port, to_timeval(connect_timeout)));
+ }
if (!m_context) {
throw Failed("Redis context construction error");
void
RedisStorageBackend::select_database(const Url& url)
{
+ std::optional<std::string> db;
+ if (url.scheme() == "redis+unix") {
+ const auto parameters_map = split_parameters(url.query());
+ auto search = parameters_map.find("db");
+ if (search != parameters_map.end()) {
+ db = search->second;
+ }
+ } else {
+ if (!url.path().empty()) {
+ db = url.path().substr(1);
+ }
+ }
const uint32_t db_number =
- url.path().empty() ? 0
- : util::value_or_throw<core::Fatal>(util::parse_unsigned(
- url.path().substr(1),
- 0,
- std::numeric_limits<uint32_t>::max(),
- "db number"));
+ !db ? 0
+ : util::value_or_throw<core::Fatal>(util::parse_unsigned(
+ *db, 0, std::numeric_limits<uint32_t>::max(), "db number"));
if (db_number != 0) {
LOG("Redis SELECT {}", db_number);
addtest(secondary_file)
addtest(secondary_http)
addtest(secondary_redis)
+addtest(secondary_redis_unix)
addtest(secondary_url)
addtest(serialize_diagnostics)
addtest(source_date_epoch)
--- /dev/null
+SUITE_secondary_redis_unix_PROBE() {
+ if ! $CCACHE --version | fgrep -q -- redis-storage &> /dev/null; then
+ echo "redis-storage not available"
+ return
+ fi
+ if ! command -v redis-server &> /dev/null; then
+ echo "redis-server not found"
+ return
+ fi
+ if redis-server --unixsocket /foo/redis.sock 2>&1 | grep "FATAL CONFIG FILE ERROR" &> /dev/null; then
+ # "Bad directive or wrong number of arguments"
+ echo "redis-server without unixsocket"
+ return
+ fi
+ if ! command -v redis-cli &> /dev/null; then
+ echo "redis-cli not found"
+ return
+ fi
+ if ! redis-cli -s /foo/redis.sock --version &> /dev/null; then
+ # "Unrecognized option or bad number of args"
+ echo "redis-cli without socket"
+ return
+ fi
+}
+
+start_redis_unix_server() {
+ local socket="$1"
+ local password="${2:-}"
+
+ redis-server --bind localhost --unixsocket "${socket}" --port 0 >/dev/null &
+ # Wait for server start.
+ i=0
+ while [ $i -lt 100 ] && ! redis-cli -s "${socket}" ping &>/dev/null; do
+ sleep 0.1
+ i=$((i + 1))
+ done
+
+ if [ -n "${password}" ]; then
+ redis-cli -s "${socket}" config set requirepass "${password}" &>/dev/null
+ fi
+}
+
+SUITE_secondary_redis_unix_SETUP() {
+ unset CCACHE_NODIRECT
+
+ generate_code 1 test.c
+}
+
+expect_number_of_redis_unix_cache_entries() {
+ local expected=$1
+ local url=$2
+ local socket=${url}
+ socket=${socket/#redis+unix:\/\//}
+ socket=${socket/#redis+unix:/}
+ socket=${socket/#*@localhost/}
+ socket=${socket/%\?*/} # remove query
+ local actual
+
+ actual=$(redis-cli -s "$socket" keys "ccache:*" 2>/dev/null | wc -l)
+ if [ "$actual" -ne "$expected" ]; then
+ test_failed_internal "Found $actual (expected $expected) entries in $url"
+ fi
+}
+
+SUITE_secondary_redis_unix() {
+ # -------------------------------------------------------------------------
+ TEST "Base case"
+
+ socket=$(mktemp)
+ redis_url="redis+unix:${socket}"
+ export CCACHE_SECONDARY_STORAGE="${redis_url}"
+
+ start_redis_unix_server "${socket}"
+ function expect_number_of_redis_cache_entries()
+ {
+ expect_number_of_redis_unix_cache_entries "$@"
+ }
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 0
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2
+ expect_number_of_redis_cache_entries 2 "$redis_url" # result + manifest
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 1
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2
+ expect_number_of_redis_cache_entries 2 "$redis_url" # result + manifest
+
+ $CCACHE -C >/dev/null
+ expect_stat files_in_cache 0
+ expect_number_of_redis_cache_entries 2 "$redis_url" # result + manifest
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 2
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2 # fetched from secondary
+ expect_number_of_redis_cache_entries 2 "$redis_url" # result + manifest
+
+ # -------------------------------------------------------------------------
+ TEST "Password"
+
+ socket=$(mktemp)
+ password=secret123
+ redis_url="redis+unix://${password}@localhost${socket}"
+ export CCACHE_SECONDARY_STORAGE="${redis_url}"
+
+ start_redis_unix_server "${socket}" "${password}"
+ function expect_number_of_redis_cache_entries()
+ {
+ expect_number_of_redis_unix_cache_entries "$@"
+ }
+
+ CCACHE_DEBUG=1 $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 0
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2
+ expect_number_of_redis_cache_entries 2 "$redis_url" # result + manifest
+ expect_not_contains test.o.*.ccache-log "${password}"
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 1
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2
+ expect_number_of_redis_cache_entries 2 "$redis_url" # result + manifest
+
+ $CCACHE -C >/dev/null
+ expect_stat files_in_cache 0
+ expect_number_of_redis_cache_entries 2 "$redis_url" # result + manifest
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 2
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2 # fetched from secondary
+ expect_number_of_redis_cache_entries 2 "$redis_url" # result + manifest
+
+ # -------------------------------------------------------------------------
+ TEST "Unreachable server"
+
+ export CCACHE_SECONDARY_STORAGE="redis+unix:///foo"
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 0
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2
+ expect_stat secondary_storage_error 1
+}
projects / thirdparty / ccache.git / commitdiff
