From: Anders Björklund Date: Tue, 5 Jul 2022 21:01:01 +0000 (+0200) Subject: feat: Support Redis over unix socket (#1064) X-Git-Tag: v4.7~163 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=ac7e7cf64880ecb457ce16cfb8c96e5e4ce9024a;p=thirdparty%2Fccache.git feat: Support Redis over unix socket (#1064) --- diff --git a/doc/MANUAL.adoc b/doc/MANUAL.adoc index 1b98de06e..4f5b00cb9 100644 --- a/doc/MANUAL.adoc +++ b/doc/MANUAL.adoc @@ -1158,6 +1158,7 @@ The default is *subdirs*. === Redis storage backend URL format: `+redis://[[USERNAME:]PASSWORD@]HOST[:PORT][/DBNUMBER]+` +Alternative: `+redis+unix://[[USERNAME:]PASSWORD@localhost]SOCKET_PATH[?db=DBNUMBER]+` This backend stores data in a https://redis.io[Redis] (or Redis-compatible) server. There are implementations for both memory-based and disk-based storage. @@ -1176,6 +1177,9 @@ Examples: * `+redis://localhost+` * `+redis://p4ssw0rd@cache.example.com:6379/0|connect-timeout=50+` +* `+redis+unix:/run/redis.sock+` +* `+redis+unix:///run/redis.sock+` +* `+redis+unix://p4ssw0rd@localhost/var/run/redis/redis-server.sock?db=0|connect-timeout=50+` Optional attributes: diff --git a/src/storage/Storage.cpp b/src/storage/Storage.cpp index 2fafad7b9..caac35fd9 100644 --- a/src/storage/Storage.cpp +++ b/src/storage/Storage.cpp @@ -54,6 +54,7 @@ const std::unordered_map()}, #ifdef HAVE_REDIS_STORAGE_BACKEND {"redis", std::make_shared()}, + {"redis+unix", std::make_shared()}, #endif }; @@ -355,13 +356,19 @@ Storage::get_secondary_storage_config_for_logging() const return util::join(configs, " "); } +static void +redact_url_for_logging(Url& url_for_logging) +{ + url_for_logging.user_info(""); +} + void Storage::add_secondary_storages() { const auto configs = parse_storage_configs(m_config.secondary_storage()); for (const auto& config : configs) { auto url_for_logging = config.params.url; - url_for_logging.user_info(""); + redact_url_for_logging(url_for_logging); const auto storage = get_storage(config.params.url); if (!storage) { throw core::Error("unknown secondary storage URL: {}", @@ -445,7 +452,7 @@ Storage::get_backend(SecondaryStorageEntry& entry, if (backend == entry.backends.end()) { auto shard_url_for_logging = shard_url; - shard_url_for_logging.user_info(""); + redact_url_for_logging(shard_url_for_logging); entry.backends.push_back( {shard_url, shard_url_for_logging.str(), {}, false}); auto shard_params = entry.config.params; diff --git a/src/storage/secondary/RedisStorage.cpp b/src/storage/secondary/RedisStorage.cpp index cd5c100fa..976386a7f 100644 --- a/src/storage/secondary/RedisStorage.cpp +++ b/src/storage/secondary/RedisStorage.cpp @@ -43,6 +43,7 @@ #endif #include +#include #include namespace storage::secondary { @@ -105,12 +106,33 @@ split_user_info(const std::string& user_info) } } +std::map +split_parameters(const Url::Query& query) +{ + std::map m; + if (!query.empty()) { + auto it = query.begin(); + auto end = query.end(); + do { + m[it->key()] = it->val(); + } while (++it != end); + } + return m; +} + RedisStorageBackend::RedisStorageBackend(const Params& params) : m_prefix("ccache"), // TODO: attribute m_context(nullptr, redisFree) { const auto& url = params.url; - ASSERT(url.scheme() == "redis"); + ASSERT(url.scheme() == "redis" || url.scheme() == "redis+unix"); + if (url.scheme() == "redis+unix" && !params.url.host().empty() + && params.url.host() != "localhost") { + throw core::Fatal(FMT( + "invalid file path \"{}\": specifying a host (\"{}\") is not supported", + params.url.str(), + params.url.host())); + } auto connect_timeout = k_default_connect_timeout; auto operation_timeout = k_default_operation_timeout; @@ -220,19 +242,27 @@ RedisStorageBackend::connect(const Url& url, const uint32_t connect_timeout, const uint32_t operation_timeout) { - const std::string host = url.host().empty() ? "localhost" : url.host(); - const uint32_t port = url.port().empty() - ? DEFAULT_PORT - : util::value_or_throw( - util::parse_unsigned(url.port(), 1, 65535, "port")); - ASSERT(url.path().empty() || url.path()[0] == '/'); - - LOG("Redis connecting to {}:{} (connect timeout {} ms)", - host, - port, - connect_timeout); - m_context.reset( - redisConnectWithTimeout(host.c_str(), port, to_timeval(connect_timeout))); + if (url.scheme() == "redis+unix") { + LOG("Redis connecting to unix://{} (connect timeout {} ms)", + url.path(), + connect_timeout); + m_context.reset(redisConnectUnixWithTimeout(url.path().c_str(), + to_timeval(connect_timeout))); + } else { + const std::string host = url.host().empty() ? "localhost" : url.host(); + const uint32_t port = + url.port().empty() ? DEFAULT_PORT + : util::value_or_throw( + util::parse_unsigned(url.port(), 1, 65535, "port")); + ASSERT(url.path().empty() || url.path()[0] == '/'); + + LOG("Redis connecting to {}:{} (connect timeout {} ms)", + host, + port, + connect_timeout); + m_context.reset( + redisConnectWithTimeout(host.c_str(), port, to_timeval(connect_timeout))); + } if (!m_context) { throw Failed("Redis context construction error"); @@ -257,13 +287,22 @@ RedisStorageBackend::connect(const Url& url, void RedisStorageBackend::select_database(const Url& url) { + std::optional db; + if (url.scheme() == "redis+unix") { + const auto parameters_map = split_parameters(url.query()); + auto search = parameters_map.find("db"); + if (search != parameters_map.end()) { + db = search->second; + } + } else { + if (!url.path().empty()) { + db = url.path().substr(1); + } + } const uint32_t db_number = - url.path().empty() ? 0 - : util::value_or_throw(util::parse_unsigned( - url.path().substr(1), - 0, - std::numeric_limits::max(), - "db number")); + !db ? 0 + : util::value_or_throw(util::parse_unsigned( + *db, 0, std::numeric_limits::max(), "db number")); if (db_number != 0) { LOG("Redis SELECT {}", db_number); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 42d4c22fe..8f80e20cd 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -57,6 +57,7 @@ addtest(sanitize_blacklist) addtest(secondary_file) addtest(secondary_http) addtest(secondary_redis) +addtest(secondary_redis_unix) addtest(secondary_url) addtest(serialize_diagnostics) addtest(source_date_epoch) diff --git a/test/suites/secondary_redis_unix.bash b/test/suites/secondary_redis_unix.bash new file mode 100644 index 000000000..e53aaf0ea --- /dev/null +++ b/test/suites/secondary_redis_unix.bash @@ -0,0 +1,148 @@ +SUITE_secondary_redis_unix_PROBE() { + if ! $CCACHE --version | fgrep -q -- redis-storage &> /dev/null; then + echo "redis-storage not available" + return + fi + if ! command -v redis-server &> /dev/null; then + echo "redis-server not found" + return + fi + if redis-server --unixsocket /foo/redis.sock 2>&1 | grep "FATAL CONFIG FILE ERROR" &> /dev/null; then + # "Bad directive or wrong number of arguments" + echo "redis-server without unixsocket" + return + fi + if ! command -v redis-cli &> /dev/null; then + echo "redis-cli not found" + return + fi + if ! redis-cli -s /foo/redis.sock --version &> /dev/null; then + # "Unrecognized option or bad number of args" + echo "redis-cli without socket" + return + fi +} + +start_redis_unix_server() { + local socket="$1" + local password="${2:-}" + + redis-server --bind localhost --unixsocket "${socket}" --port 0 >/dev/null & + # Wait for server start. + i=0 + while [ $i -lt 100 ] && ! redis-cli -s "${socket}" ping &>/dev/null; do + sleep 0.1 + i=$((i + 1)) + done + + if [ -n "${password}" ]; then + redis-cli -s "${socket}" config set requirepass "${password}" &>/dev/null + fi +} + +SUITE_secondary_redis_unix_SETUP() { + unset CCACHE_NODIRECT + + generate_code 1 test.c +} + +expect_number_of_redis_unix_cache_entries() { + local expected=$1 + local url=$2 + local socket=${url} + socket=${socket/#redis+unix:\/\//} + socket=${socket/#redis+unix:/} + socket=${socket/#*@localhost/} + socket=${socket/%\?*/} # remove query + local actual + + actual=$(redis-cli -s "$socket" keys "ccache:*" 2>/dev/null | wc -l) + if [ "$actual" -ne "$expected" ]; then + test_failed_internal "Found $actual (expected $expected) entries in $url" + fi +} + +SUITE_secondary_redis_unix() { + # ------------------------------------------------------------------------- + TEST "Base case" + + socket=$(mktemp) + redis_url="redis+unix:${socket}" + export CCACHE_SECONDARY_STORAGE="${redis_url}" + + start_redis_unix_server "${socket}" + function expect_number_of_redis_cache_entries() + { + expect_number_of_redis_unix_cache_entries "$@" + } + + $CCACHE_COMPILE -c test.c + expect_stat direct_cache_hit 0 + expect_stat cache_miss 1 + expect_stat files_in_cache 2 + expect_number_of_redis_cache_entries 2 "$redis_url" # result + manifest + + $CCACHE_COMPILE -c test.c + expect_stat direct_cache_hit 1 + expect_stat cache_miss 1 + expect_stat files_in_cache 2 + expect_number_of_redis_cache_entries 2 "$redis_url" # result + manifest + + $CCACHE -C >/dev/null + expect_stat files_in_cache 0 + expect_number_of_redis_cache_entries 2 "$redis_url" # result + manifest + + $CCACHE_COMPILE -c test.c + expect_stat direct_cache_hit 2 + expect_stat cache_miss 1 + expect_stat files_in_cache 2 # fetched from secondary + expect_number_of_redis_cache_entries 2 "$redis_url" # result + manifest + + # ------------------------------------------------------------------------- + TEST "Password" + + socket=$(mktemp) + password=secret123 + redis_url="redis+unix://${password}@localhost${socket}" + export CCACHE_SECONDARY_STORAGE="${redis_url}" + + start_redis_unix_server "${socket}" "${password}" + function expect_number_of_redis_cache_entries() + { + expect_number_of_redis_unix_cache_entries "$@" + } + + CCACHE_DEBUG=1 $CCACHE_COMPILE -c test.c + expect_stat direct_cache_hit 0 + expect_stat cache_miss 1 + expect_stat files_in_cache 2 + expect_number_of_redis_cache_entries 2 "$redis_url" # result + manifest + expect_not_contains test.o.*.ccache-log "${password}" + + $CCACHE_COMPILE -c test.c + expect_stat direct_cache_hit 1 + expect_stat cache_miss 1 + expect_stat files_in_cache 2 + expect_number_of_redis_cache_entries 2 "$redis_url" # result + manifest + + $CCACHE -C >/dev/null + expect_stat files_in_cache 0 + expect_number_of_redis_cache_entries 2 "$redis_url" # result + manifest + + $CCACHE_COMPILE -c test.c + expect_stat direct_cache_hit 2 + expect_stat cache_miss 1 + expect_stat files_in_cache 2 # fetched from secondary + expect_number_of_redis_cache_entries 2 "$redis_url" # result + manifest + + # ------------------------------------------------------------------------- + TEST "Unreachable server" + + export CCACHE_SECONDARY_STORAGE="redis+unix:///foo" + + $CCACHE_COMPILE -c test.c + expect_stat direct_cache_hit 0 + expect_stat cache_miss 1 + expect_stat files_in_cache 2 + expect_stat secondary_storage_error 1 +}