From: Anders Björklund Date: Sat, 10 Jul 2021 18:39:59 +0000 (+0200) Subject: Add optional Redis secondary storage backend (#875) X-Git-Tag: v4.4~141 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=1ef63482fb1fd8508dae311a690442b43f656be6;p=thirdparty%2Fccache.git Add optional Redis secondary storage backend (#875) --- diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 0de2cde97..fef0dbfad 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -93,9 +93,9 @@ jobs: # Install ld.gold (binutils) and ld.lld on different runs. if [ "${{ matrix.config.os }}" = "ubuntu-18.04" ]; then - sudo apt-get install -y ninja-build elfutils libzstd-dev binutils python3 + sudo apt-get install -y ninja-build elfutils libzstd-dev pkg-config libhiredis-dev redis-server redis-tools binutils python3 else - sudo apt-get install -y ninja-build elfutils libzstd-dev lld python3 + sudo apt-get install -y ninja-build elfutils libzstd-dev pkg-config libhiredis-dev redis-server redis-tools lld python3 fi if [ "${{ matrix.config.compiler }}" = "gcc" ]; then @@ -111,7 +111,7 @@ jobs: fi elif [ "${{ runner.os }}" = "macOS" ]; then HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1 \ - brew install ninja + brew install ninja pkg-config hiredis redis if [ "${{ matrix.config.compiler }}" = "gcc" ]; then brew install gcc@${{ matrix.config.version }} @@ -159,7 +159,7 @@ jobs: BUILDDIR: . CCACHE_LOC: . CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=Debug -DENABLE_TRACING=1 - apt_get: elfutils libzstd-dev + apt_get: elfutils libzstd-dev pkg-config libhiredis-dev - name: Linux GCC 32-bit os: ubuntu-18.04 @@ -168,7 +168,7 @@ jobs: CFLAGS: -m32 -g -O2 CXXFLAGS: -m32 -g -O2 LDFLAGS: -m32 - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DREDIS_STORAGE_BACKEND=OFF ENABLE_CACHE_CLEANUP_TESTS: 1 apt_get: elfutils gcc-multilib g++-multilib lib32stdc++-5-dev @@ -176,7 +176,7 @@ jobs: os: ubuntu-18.04 CC: gcc CXX: g++ - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DREDIS_STORAGE_BACKEND=OFF ENABLE_CACHE_CLEANUP_TESTS: 1 CUDA: 10.1.243-1 apt_get: elfutils libzstd-dev @@ -185,7 +185,7 @@ jobs: os: ubuntu-18.04 CC: i686-w64-mingw32-gcc-posix CXX: i686-w64-mingw32-g++-posix - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DCMAKE_SYSTEM_NAME=Windows -DZSTD_FROM_INTERNET=ON + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DCMAKE_SYSTEM_NAME=Windows -DZSTD_FROM_INTERNET=ON -DREDIS_STORAGE_BACKEND=OFF RUN_TESTS: none apt_get: elfutils mingw-w64 @@ -194,7 +194,7 @@ jobs: CC: x86_64-w64-mingw32-gcc-posix CXX: x86_64-w64-mingw32-g++-posix ENABLE_CACHE_CLEANUP_TESTS: 1 - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DCMAKE_SYSTEM_NAME=Windows -DZSTD_FROM_INTERNET=ON + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DCMAKE_SYSTEM_NAME=Windows -DZSTD_FROM_INTERNET=ON -DREDIS_STORAGE_BACKEND=OFF RUN_TESTS: unittest-in-wine apt_get: elfutils mingw-w64 wine @@ -206,7 +206,7 @@ jobs: CXX: cl ENABLE_CACHE_CLEANUP_TESTS: 1 CMAKE_GENERATOR: Ninja - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DREDIS_STORAGE_BACKEND=OFF TEST_CC: clang -target i686-pc-windows-msvc - name: Windows VS2019 64-bit @@ -217,7 +217,7 @@ jobs: CXX: cl ENABLE_CACHE_CLEANUP_TESTS: 1 CMAKE_GENERATOR: Ninja - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DREDIS_STORAGE_BACKEND=OFF TEST_CC: clang -target x86_64-pc-windows-msvc - name: Clang address & UB sanitizer @@ -227,7 +227,7 @@ jobs: ENABLE_CACHE_CLEANUP_TESTS: 1 CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DENABLE_SANITIZER_ADDRESS=ON -DENABLE_SANITIZER_UNDEFINED_BEHAVIOR=ON ASAN_OPTIONS: detect_leaks=0 - apt_get: elfutils libzstd-dev + apt_get: elfutils libzstd-dev pkg-config libhiredis-dev - name: Clang static analyzer os: ubuntu-20.04 @@ -236,7 +236,7 @@ jobs: ENABLE_CACHE_CLEANUP_TESTS: 1 CMAKE_PREFIX: scan-build RUN_TESTS: none - apt_get: libzstd-dev + apt_get: libzstd-dev pkg-config libhiredis-dev - name: Linux binary os: ubuntu-20.04 @@ -244,26 +244,26 @@ jobs: CXX: g++ SPECIAL: build-and-verify-package CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=Release - apt_get: elfutils libzstd-dev ninja-build + apt_get: elfutils libzstd-dev pkg-config libhiredis-dev ninja-build - name: Source package os: ubuntu-20.04 CC: gcc CXX: g++ SPECIAL: build-and-verify-source-package - apt_get: elfutils libzstd-dev ninja-build asciidoc xsltproc docbook-xml docbook-xsl + apt_get: elfutils libzstd-dev pkg-config libhiredis-dev ninja-build asciidoc xsltproc docbook-xml docbook-xsl - name: HTML documentation os: ubuntu-18.04 EXTRA_CMAKE_BUILD_FLAGS: --target doc-html RUN_TESTS: none - apt_get: libzstd-dev asciidoc docbook-xml docbook-xsl + apt_get: libzstd-dev pkg-config libhiredis-dev asciidoc docbook-xml docbook-xsl - name: Manual page os: ubuntu-18.04 EXTRA_CMAKE_BUILD_FLAGS: --target doc-man-page RUN_TESTS: none - apt_get: libzstd-dev asciidoc xsltproc docbook-xml docbook-xsl + apt_get: libzstd-dev pkg-config libhiredis-dev asciidoc xsltproc docbook-xml docbook-xsl - name: Clang-Tidy os: ubuntu-18.04 @@ -271,7 +271,7 @@ jobs: CXX: clang++-9 RUN_TESTS: none CMAKE_PARAMS: -DENABLE_CLANG_TIDY=ON -DCLANGTIDY=/usr/bin/clang-tidy-9 - apt_get: libzstd-dev clang-9 clang-tidy-9 + apt_get: libzstd-dev pkg-config libhiredis-dev clang-9 clang-tidy-9 steps: - name: Get source diff --git a/.github/workflows/codeql-analysis.yaml b/.github/workflows/codeql-analysis.yaml index b19e80763..e159d92e6 100644 --- a/.github/workflows/codeql-analysis.yaml +++ b/.github/workflows/codeql-analysis.yaml @@ -31,7 +31,7 @@ jobs: fetch-depth: 2 - name: Install dependencies - run: sudo apt-get update && sudo apt-get install ninja-build elfutils libzstd-dev + run: sudo apt-get update && sudo apt-get install ninja-build elfutils libzstd-dev pkg-config libhiredis-dev - name: Initialize CodeQL uses: github/codeql-action/init@v1 diff --git a/CMakeLists.txt b/CMakeLists.txt index d37342296..f06f0468b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -97,6 +97,11 @@ endif() option(ZSTD_FROM_INTERNET "Download and use libzstd from the Internet" ${ZSTD_FROM_INTERNET_DEFAULT}) find_package(zstd 1.1.2 REQUIRED) +option(REDIS_STORAGE_BACKEND "Enable Redis secondary storage" ON) +if(REDIS_STORAGE_BACKEND) + find_package(hiredis 0.13.3 REQUIRED) +endif() + # # Special flags # diff --git a/cmake/Findhiredis.cmake b/cmake/Findhiredis.cmake new file mode 100644 index 000000000..5667d32d4 --- /dev/null +++ b/cmake/Findhiredis.cmake @@ -0,0 +1,30 @@ +find_package(PkgConfig) +if(PKG_CONFIG_FOUND) + pkg_check_modules(HIREDIS REQUIRED hiredis>=${hiredis_FIND_VERSION}) + find_library(HIREDIS_LIBRARY ${HIREDIS_LIBRARIES} HINTS ${HIREDIS_LIBDIR}) + find_path(HIREDIS_INCLUDE_DIR hiredis/hiredis.h HINTS ${HIREDIS_PREFIX}/include) +else() + find_library(HIREDIS_LIBRARY hiredis) + find_path(HIREDIS_INCLUDE_DIR hiredis/hiredis.h) +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + hiredis "please install libhiredis" + HIREDIS_INCLUDE_DIR HIREDIS_LIBRARY) +mark_as_advanced(HIREDIS_INCLUDE_DIR HIREDIS_LIBRARY) + +add_library(HIREDIS::HIREDIS UNKNOWN IMPORTED) +set_target_properties( + HIREDIS::HIREDIS + PROPERTIES + IMPORTED_LOCATION "${HIREDIS_LIBRARY}" + INTERFACE_COMPILE_OPTIONS "${HIREDIS_CFLAGS_OTHER}" + INTERFACE_INCLUDE_DIRECTORIES "${HIREDIS_INCLUDE_DIR}") + +include(FeatureSummary) +set_package_properties( + hiredis + PROPERTIES + URL "https://github.com/redis/hiredis" + DESCRIPTION "Hiredis is a minimalistic C client library for the Redis database") diff --git a/doc/MANUAL.adoc b/doc/MANUAL.adoc index f7386d3e3..09392d760 100644 --- a/doc/MANUAL.adoc +++ b/doc/MANUAL.adoc @@ -797,6 +797,7 @@ Examples: * `file:///shared/nfs/directory` * `file:///shared/nfs/one|read-only file:///shared/nfs/two` * `http://example.org/cache` +* `redis://server.example.com` [[config_sloppiness]] *sloppiness* (*CCACHE_SLOPPINESS*):: @@ -953,6 +954,30 @@ Known issues and limitations: * HTTPS is not yet supported. * URLs containing IPv6 addresses like `http://[::1]/` are not supported. +=== Redis storage backend + +URL format: `redis://HOST[:PORT]` or `redis://UNIX_SOCKET_PATH` + +This backend stores data in a Redis (or Redis-compatible) server. +There are implementations for both memory and disk based storage. + +See + +Note that ccache will not perform any cleanup of the Redis storage. +But you can configure LRU eviction: + +Examples: + +* `redis://localhost:6379` +* `redis:///var/run/redis/redis-server.sock` + +Optional attributes: + +* *connect-timeout*: Timeout (in ms) for network connection. +* *operation-timeout*: Timeout (in ms) for Redis commands. +* *username*: Username for the Redis AUTH command. (Requires ACL) +* *password*: Password for the Redis AUTH command. + == Cache size management By default, ccache has a 5 GB limit on the total size of files in the cache and diff --git a/misc/upload-redis b/misc/upload-redis new file mode 100755 index 000000000..6fe4335fb --- /dev/null +++ b/misc/upload-redis @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 + +# this script will upload the contents of the cache, +# from primary storage to the redis secondary storage + +import redis +import os + +config = os.getenv("REDIS_CONF", "localhost") +if ":" in config: + host, port = config.rsplit(":", 1) + sock = None +elif config.startswith("/"): + host, port, sock = None, None, config +else: + host, port, sock = config, 6379, None +username = os.getenv("REDIS_USERNAME") +password = os.getenv("REDIS_PASSWORD") +context = redis.Redis(host=host, port=port, unix_socket_path=sock, password=password) + +CCACHE_MANIFEST = b"cCmF" +CCACHE_RESULT = b"cCrS" + +ccache = os.getenv("CCACHE_DIR", os.path.expanduser("~/.cache/ccache")) +filelist = [] +for dirpath, dirnames, filenames in os.walk(ccache): + # sort by modification time, most recently used last + for filename in filenames: + if filename.endswith(".lock"): + continue + stat = os.stat(os.path.join(dirpath, filename)) + filelist.append((stat.st_mtime, dirpath, filename)) +filelist.sort() +files = result = manifest = objects = 0 +for mtime, dirpath, filename in filelist: + dirname = dirpath.replace(ccache + os.path.sep, "") + if dirname == "tmp": + continue + elif filename == "CACHEDIR.TAG" or filename == "stats": + # ignore these + files = files + 1 + else: + (base, ext) = filename[:-1], filename[-1:] + if ext == "R" or ext == "M": + if ext == "R": + result = result + 1 + if ext == "M": + manifest = manifest + 1 + key = "ccache:" + "".join(list(os.path.split(dirname)) + [base]) + val = open(os.path.join(dirpath, filename), "rb").read() or None + if val: + print("%s: %s %d" % (key, ext, len(val))) + magic = val[0:4] + if ext == "M": + assert magic == CCACHE_MANIFEST + if ext == "R": + assert magic == CCACHE_RESULT + context.set(key, val) + objects = objects + 1 + files = files + 1 +print( + "%d files, %d result (%d manifest) = %d objects" + % (files, result, manifest, objects) +) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9b60b2109..168234b00 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -84,6 +84,14 @@ target_link_libraries( target_include_directories(ccache_lib PRIVATE ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) +if(REDIS_STORAGE_BACKEND) + target_compile_definitions(ccache_lib PRIVATE -DHAVE_REDIS_STORAGE_BACKEND) + target_link_libraries( + ccache_lib + PUBLIC standard_settings standard_warnings HIREDIS::HIREDIS + third_party_lib) +endif() + add_subdirectory(core) add_subdirectory(storage) add_subdirectory(third_party) diff --git a/src/storage/Storage.cpp b/src/storage/Storage.cpp index fabe34d34..e0224f731 100644 --- a/src/storage/Storage.cpp +++ b/src/storage/Storage.cpp @@ -26,6 +26,9 @@ #include #include #include +#ifdef HAVE_REDIS_STORAGE_BACKEND +# include +#endif #include #include @@ -245,6 +248,13 @@ create_storage(const ParseStorageEntryResult& storage_entry) storage_entry.attributes); } +#ifdef HAVE_REDIS_STORAGE_BACKEND + if (storage_entry.url.scheme() == "redis") { + return std::make_unique(storage_entry.url, + storage_entry.attributes); + } +#endif + return {}; } diff --git a/src/storage/secondary/CMakeLists.txt b/src/storage/secondary/CMakeLists.txt index f2292245c..83f0825e8 100644 --- a/src/storage/secondary/CMakeLists.txt +++ b/src/storage/secondary/CMakeLists.txt @@ -4,4 +4,8 @@ set( ${CMAKE_CURRENT_SOURCE_DIR}/HttpStorage.cpp ) +if(REDIS_STORAGE_BACKEND) + list(APPEND sources ${CMAKE_CURRENT_SOURCE_DIR}/RedisStorage.cpp) +endif() + target_sources(ccache_lib PRIVATE ${sources}) diff --git a/src/storage/secondary/RedisStorage.cpp b/src/storage/secondary/RedisStorage.cpp new file mode 100644 index 000000000..4dd9eb0b5 --- /dev/null +++ b/src/storage/secondary/RedisStorage.cpp @@ -0,0 +1,352 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "RedisStorage.hpp" + +#include +#include +#include + +#include + +namespace storage { +namespace secondary { + +const struct timeval DEFAULT_CONNECT_TIMEOUT = {0, 100 * 1000}; // 100 ms +const struct timeval DEFAULT_OPERATION_TIMEOUT = {10, 0 * 1000}; // 10 sec + +static struct timeval +milliseconds_to_timeval(const std::string& msec) +{ + int ms = std::stoi(msec); + struct timeval tv; + tv.tv_sec = ms / 1000; + tv.tv_usec = (ms % 1000) * 1000; + return tv; +} + +static std::string +timeval_to_string(struct timeval tv) +{ + return FMT("{:.3f}s", tv.tv_sec + tv.tv_usec / 1000000.0); +} + +static nonstd::optional +parse_timeout_attribute(const AttributeMap& attributes, const std::string& name) +{ + const auto it = attributes.find(name); + if (it == attributes.end()) { + return nonstd::nullopt; + } + return milliseconds_to_timeval(it->second); +} + +static nonstd::optional +parse_string_attribute(const AttributeMap& attributes, const std::string& name) +{ + const auto it = attributes.find(name); + if (it == attributes.end()) { + return nonstd::nullopt; + } + return it->second; +} + +RedisStorage::RedisStorage(const Url& url, const AttributeMap& attributes) + : m_url(url), + m_connect_timeout(parse_timeout_attribute(attributes, "connect-timeout")), + m_operation_timeout( + parse_timeout_attribute(attributes, "operation-timeout")), + m_username(parse_string_attribute(attributes, "username")), + m_password(parse_string_attribute(attributes, "password")) +{ + m_prefix = "ccache"; // TODO: attribute + m_context = nullptr; + m_connected = false; + m_invalid = false; +} + +RedisStorage::~RedisStorage() +{ + disconnect(); + if (m_context) { + redisFree(m_context); + m_context = nullptr; + } +} + +int +RedisStorage::connect() +{ + if (m_connected) { + return REDIS_OK; + } + if (m_invalid) { + return REDIS_ERR; + } + + if (m_context) { + if (redisReconnect(m_context) == REDIS_OK) { + m_connected = true; + return REDIS_OK; + } + LOG("Redis reconnect err: {}", m_context->errstr); + redisFree(m_context); + m_context = nullptr; + } + + ASSERT(m_url.scheme() == "redis"); + std::string host = m_url.host(); + std::string port = m_url.port(); + std::string sock = m_url.path(); + if (m_connect_timeout) { + LOG("Redis connect timeout {}", timeval_to_string(*m_connect_timeout)); + } + struct timeval connect_timeout = + m_connect_timeout ? *m_connect_timeout : DEFAULT_CONNECT_TIMEOUT; + if (!host.empty()) { + int p = port.empty() ? 6379 : std::stoi(port); + m_context = redisConnectWithTimeout(host.c_str(), p, connect_timeout); + } else if (!sock.empty()) { + m_context = redisConnectUnixWithTimeout(sock.c_str(), connect_timeout); + } else { + LOG("Redis invalid url: {}", m_url.str()); + m_invalid = true; + return REDIS_ERR; + } + + if (!m_context) { + LOG("Redis connect {} err NULL", m_url.str()); + m_invalid = true; + return REDIS_ERR; + } else if (m_context->err) { + LOG("Redis connect {} err: {}", m_url.str(), m_context->errstr); + m_invalid = true; + return m_context->err; + } else { + if (m_context->connection_type == REDIS_CONN_TCP) { + LOG( + "Redis connect tcp {}:{} OK", m_context->tcp.host, m_context->tcp.port); + } + if (m_context->connection_type == REDIS_CONN_UNIX) { + LOG("Redis connect unix {} OK", m_context->unix_sock.path); + } + m_connected = true; + + if (m_operation_timeout) { + LOG("Redis timeout {}", timeval_to_string(*m_operation_timeout)); + } + struct timeval operation_timeout = + m_operation_timeout ? *m_operation_timeout : DEFAULT_OPERATION_TIMEOUT; + if (redisSetTimeout(m_context, operation_timeout) != REDIS_OK) { + LOG_RAW("Failed to set timeout"); + } + + return auth(); + } +} + +int +RedisStorage::auth() +{ + if (m_password) { + bool log_password = false; + std::string username = m_username ? *m_username : "default"; + std::string password = log_password ? *m_password : "*******"; + LOG("Redis AUTH {} {}", username, password); + redisReply* reply; + if (m_username) { + reply = static_cast(redisCommand( + m_context, "AUTH %s %s", m_username->c_str(), m_password->c_str())); + } else { + reply = static_cast( + redisCommand(m_context, "AUTH %s", m_password->c_str())); + } + if (!reply) { + LOG("Failed to auth {} in redis", username); + m_invalid = true; + } else if (reply->type == REDIS_REPLY_ERROR) { + LOG("Failed to auth {} in redis: {}", username, reply->str); + m_invalid = true; + } + freeReplyObject(reply); + if (m_invalid) { + return REDIS_ERR; + } + } + return REDIS_OK; +} + +inline bool +is_error(int err) +{ + return (err != REDIS_OK); +} + +inline bool +is_timeout(int err) +{ +#ifdef REDIS_ERR_TIMEOUT + // Only returned for hiredis version 1.0.0 and above + return (err == REDIS_ERR_TIMEOUT); +#else + (void)err; + return false; +#endif +} + +void +RedisStorage::disconnect() +{ + if (m_connected) { + // Note: only the async API actually disconnects from the server + // the connection is eventually cleaned up in redisFree() + LOG_RAW("Redis disconnect"); + m_connected = false; + } +} + +nonstd::expected, SecondaryStorage::Error> +RedisStorage::get(const Digest& key) +{ + int err = connect(); + if (is_timeout(err)) { + return nonstd::make_unexpected(Error::timeout); + } else if (is_error(err)) { + return nonstd::make_unexpected(Error::error); + } + const std::string key_string = get_key_string(key); + LOG("Redis GET {}", key_string); + redisReply* reply = static_cast( + redisCommand(m_context, "GET %s", key_string.c_str())); + bool found = false; + bool missing = false; + std::string value; + if (!reply) { + LOG("Failed to get {} from redis", key_string); + } else if (reply->type == REDIS_REPLY_ERROR) { + LOG("Failed to get {} from redis: {}", key_string, reply->str); + } else if (reply->type == REDIS_REPLY_STRING) { + value = std::string(reply->str, reply->len); + found = true; + } else if (reply->type == REDIS_REPLY_NIL) { + missing = true; + } + freeReplyObject(reply); + if (found) { + return value; + } else if (missing) { + return nonstd::nullopt; + } else { + return nonstd::make_unexpected(Error::error); + } +} + +nonstd::expected +RedisStorage::put(const Digest& key, + const std::string& value, + bool only_if_missing) +{ + int err = connect(); + if (is_timeout(err)) { + return nonstd::make_unexpected(Error::timeout); + } else if (is_error(err)) { + return nonstd::make_unexpected(Error::error); + } + const std::string key_string = get_key_string(key); + if (only_if_missing) { + LOG("Redis EXISTS {}", key_string); + redisReply* reply = static_cast( + redisCommand(m_context, "EXISTS %s", key_string.c_str())); + int count = 0; + if (!reply) { + LOG("Failed to check {} in redis", key_string); + } else if (reply->type == REDIS_REPLY_ERROR) { + LOG("Failed to check {} in redis: {}", key_string, reply->str); + } else if (reply->type == REDIS_REPLY_INTEGER) { + count = reply->integer; + } + freeReplyObject(reply); + if (count > 0) { + return false; + } + } + LOG("Redis SET {}", key_string); + redisReply* reply = static_cast(redisCommand( + m_context, "SET %s %b", key_string.c_str(), value.data(), value.size())); + bool stored = false; + if (!reply) { + LOG("Failed to set {} to redis", key_string); + } else if (reply->type == REDIS_REPLY_ERROR) { + LOG("Failed to set {} to redis: {}", key_string, reply->str); + } else if (reply->type == REDIS_REPLY_STATUS) { + stored = true; + } else { + LOG("Failed to set {} to redis: {}", key_string, reply->type); + } + freeReplyObject(reply); + if (stored) { + return true; + } else { + return nonstd::make_unexpected(Error::error); + } +} + +nonstd::expected +RedisStorage::remove(const Digest& key) +{ + int err = connect(); + if (is_timeout(err)) { + return nonstd::make_unexpected(Error::timeout); + } else if (is_error(err)) { + return nonstd::make_unexpected(Error::error); + } + const std::string key_string = get_key_string(key); + LOG("Redis DEL {}", key_string); + redisReply* reply = static_cast( + redisCommand(m_context, "DEL %s", key_string.c_str())); + bool removed = false; + bool missing = false; + if (!reply) { + LOG("Failed to del {} in redis", key_string); + } else if (reply->type == REDIS_REPLY_ERROR) { + LOG("Failed to del {} in redis: {}", key_string, reply->str); + } else if (reply->type == REDIS_REPLY_INTEGER) { + if (reply->integer > 0) { + removed = true; + } else { + missing = true; + } + } + freeReplyObject(reply); + if (removed) { + return true; + } else if (missing) { + return false; + } else { + return nonstd::make_unexpected(Error::error); + } +} + +std::string +RedisStorage::get_key_string(const Digest& digest) const +{ + return FMT("{}:{}", m_prefix, digest.to_string()); +} + +} // namespace secondary +} // namespace storage diff --git a/src/storage/secondary/RedisStorage.hpp b/src/storage/secondary/RedisStorage.hpp new file mode 100644 index 000000000..2a12ea1fa --- /dev/null +++ b/src/storage/secondary/RedisStorage.hpp @@ -0,0 +1,66 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include "storage/SecondaryStorage.hpp" +#include "storage/types.hpp" + +#include + +#ifdef HAVE_SYS_TIME_H +# include +#endif + +struct redisContext; + +namespace storage { +namespace secondary { + +class RedisStorage : public storage::SecondaryStorage +{ +public: + RedisStorage(const Url& url, const AttributeMap& attributes); + ~RedisStorage(); + + nonstd::expected, Error> + get(const Digest& key) override; + nonstd::expected put(const Digest& key, + const std::string& value, + bool only_if_missing) override; + nonstd::expected remove(const Digest& key) override; + +private: + Url m_url; + std::string m_prefix; + redisContext* m_context; + const nonstd::optional m_connect_timeout; + const nonstd::optional m_operation_timeout; + const nonstd::optional m_username; + const nonstd::optional m_password; + bool m_connected; + bool m_invalid; + + int connect(); + int auth(); + void disconnect(); + std::string get_key_string(const Digest& digest) const; +}; + +} // namespace secondary +} // namespace storage diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 1e92f3489..20ccabfdc 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -54,6 +54,7 @@ addtest(readonly_direct) addtest(sanitize_blacklist) addtest(secondary_file) addtest(secondary_http) +addtest(secondary_redis) addtest(secondary_url) addtest(serialize_diagnostics) addtest(source_date_epoch) diff --git a/test/suites/secondary_redis.bash b/test/suites/secondary_redis.bash new file mode 100644 index 000000000..6eb568eae --- /dev/null +++ b/test/suites/secondary_redis.bash @@ -0,0 +1,95 @@ +SUITE_secondary_redis_PROBE() { + if ! command -v redis-server &> /dev/null; then + echo "redis-server not found" + return + fi + if ! command -v redis-cli &> /dev/null; then + echo "redis-cli not found" + return + fi +} + +SUITE_secondary_redis_SETUP() { + unset CCACHE_NODIRECT + export CCACHE_SECONDARY_STORAGE="redis://localhost:7777" + + generate_code 1 test.c +} + +expect_number_of_cache_entries() { + local expected=$1 + local url=$2 + local actual + + actual=$(redis-cli -u "$url" keys "ccache:*" | wc -l) + if [ "$actual" -ne "$expected" ]; then + test_failed_internal "Found $actual (expected $expected) entries in $url" + fi +} + +SUITE_secondary_redis() { + if $HOST_OS_APPLE; then + # no coreutils on darwin by default, perl rather than gtimeout + function timeout() { perl -e 'alarm shift; exec @ARGV' "$@"; } + fi + timeout 10 redis-server --bind localhost --port 7777 & + sleep 0.1 # wait for boot + redis-cli -p 7777 ping + + secondary="redis://localhost:7777" + + # ------------------------------------------------------------------------- + TEST "Base case" + + $CCACHE_COMPILE -c test.c + expect_stat 'cache hit (direct)' 0 + expect_stat 'cache miss' 1 + expect_stat 'files in cache' 2 + expect_number_of_cache_entries 2 "$secondary" # result + manifest + + $CCACHE_COMPILE -c test.c + expect_stat 'cache hit (direct)' 1 + expect_stat 'cache miss' 1 + expect_stat 'files in cache' 2 + expect_number_of_cache_entries 2 "$secondary" # result + manifest + + $CCACHE -C >/dev/null + expect_stat 'files in cache' 0 + expect_number_of_cache_entries 2 "$secondary" # result + manifest + + $CCACHE_COMPILE -c test.c + expect_stat 'cache hit (direct)' 2 + expect_stat 'cache miss' 1 + expect_stat 'files in cache' 0 + expect_number_of_cache_entries 2 "$secondary" # result + manifest + + redis-cli -p 7777 flushdb + + # ------------------------------------------------------------------------- + TEST "Read-only" + + $CCACHE_COMPILE -c test.c + expect_stat 'cache hit (direct)' 0 + expect_stat 'cache miss' 1 + expect_stat 'files in cache' 2 + expect_number_of_cache_entries 2 "$secondary" # result + manifest + + $CCACHE -C >/dev/null + expect_stat 'files in cache' 0 + expect_number_of_cache_entries 2 "$secondary" # result + manifest + + CCACHE_SECONDARY_STORAGE+="|read-only" + + $CCACHE_COMPILE -c test.c + expect_stat 'cache hit (direct)' 1 + expect_stat 'cache miss' 1 + expect_stat 'files in cache' 0 + expect_number_of_cache_entries 2 "$secondary" # result + manifest + + echo 'int x;' >> test.c + $CCACHE_COMPILE -c test.c + expect_stat 'cache hit (direct)' 1 + expect_stat 'cache miss' 2 + expect_stat 'files in cache' 2 + expect_number_of_cache_entries 2 "$secondary" # result + manifest +}