# Install ld.gold (binutils) and ld.lld on different runs.
if [ "${{ matrix.config.os }}" = "ubuntu-18.04" ]; then
- sudo apt-get install -y ninja-build elfutils libzstd-dev binutils python3
+ sudo apt-get install -y ninja-build elfutils libzstd-dev pkg-config libhiredis-dev redis-server redis-tools binutils python3
else
- sudo apt-get install -y ninja-build elfutils libzstd-dev lld python3
+ sudo apt-get install -y ninja-build elfutils libzstd-dev pkg-config libhiredis-dev redis-server redis-tools lld python3
fi
if [ "${{ matrix.config.compiler }}" = "gcc" ]; then
fi
elif [ "${{ runner.os }}" = "macOS" ]; then
HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1 \
- brew install ninja
+ brew install ninja pkg-config hiredis redis
if [ "${{ matrix.config.compiler }}" = "gcc" ]; then
brew install gcc@${{ matrix.config.version }}
BUILDDIR: .
CCACHE_LOC: .
CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=Debug -DENABLE_TRACING=1
- apt_get: elfutils libzstd-dev
+ apt_get: elfutils libzstd-dev pkg-config libhiredis-dev
- name: Linux GCC 32-bit
os: ubuntu-18.04
CFLAGS: -m32 -g -O2
CXXFLAGS: -m32 -g -O2
LDFLAGS: -m32
- CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON
+ CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DREDIS_STORAGE_BACKEND=OFF
ENABLE_CACHE_CLEANUP_TESTS: 1
apt_get: elfutils gcc-multilib g++-multilib lib32stdc++-5-dev
os: ubuntu-18.04
CC: gcc
CXX: g++
- CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON
+ CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DREDIS_STORAGE_BACKEND=OFF
ENABLE_CACHE_CLEANUP_TESTS: 1
CUDA: 10.1.243-1
apt_get: elfutils libzstd-dev
os: ubuntu-18.04
CC: i686-w64-mingw32-gcc-posix
CXX: i686-w64-mingw32-g++-posix
- CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DCMAKE_SYSTEM_NAME=Windows -DZSTD_FROM_INTERNET=ON
+ CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DCMAKE_SYSTEM_NAME=Windows -DZSTD_FROM_INTERNET=ON -DREDIS_STORAGE_BACKEND=OFF
RUN_TESTS: none
apt_get: elfutils mingw-w64
CC: x86_64-w64-mingw32-gcc-posix
CXX: x86_64-w64-mingw32-g++-posix
ENABLE_CACHE_CLEANUP_TESTS: 1
- CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DCMAKE_SYSTEM_NAME=Windows -DZSTD_FROM_INTERNET=ON
+ CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DCMAKE_SYSTEM_NAME=Windows -DZSTD_FROM_INTERNET=ON -DREDIS_STORAGE_BACKEND=OFF
RUN_TESTS: unittest-in-wine
apt_get: elfutils mingw-w64 wine
CXX: cl
ENABLE_CACHE_CLEANUP_TESTS: 1
CMAKE_GENERATOR: Ninja
- CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON
+ CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DREDIS_STORAGE_BACKEND=OFF
TEST_CC: clang -target i686-pc-windows-msvc
- name: Windows VS2019 64-bit
CXX: cl
ENABLE_CACHE_CLEANUP_TESTS: 1
CMAKE_GENERATOR: Ninja
- CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON
+ CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DREDIS_STORAGE_BACKEND=OFF
TEST_CC: clang -target x86_64-pc-windows-msvc
- name: Clang address & UB sanitizer
ENABLE_CACHE_CLEANUP_TESTS: 1
CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DENABLE_SANITIZER_ADDRESS=ON -DENABLE_SANITIZER_UNDEFINED_BEHAVIOR=ON
ASAN_OPTIONS: detect_leaks=0
- apt_get: elfutils libzstd-dev
+ apt_get: elfutils libzstd-dev pkg-config libhiredis-dev
- name: Clang static analyzer
os: ubuntu-20.04
ENABLE_CACHE_CLEANUP_TESTS: 1
CMAKE_PREFIX: scan-build
RUN_TESTS: none
- apt_get: libzstd-dev
+ apt_get: libzstd-dev pkg-config libhiredis-dev
- name: Linux binary
os: ubuntu-20.04
CXX: g++
SPECIAL: build-and-verify-package
CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=Release
- apt_get: elfutils libzstd-dev ninja-build
+ apt_get: elfutils libzstd-dev pkg-config libhiredis-dev ninja-build
- name: Source package
os: ubuntu-20.04
CC: gcc
CXX: g++
SPECIAL: build-and-verify-source-package
- apt_get: elfutils libzstd-dev ninja-build asciidoc xsltproc docbook-xml docbook-xsl
+ apt_get: elfutils libzstd-dev pkg-config libhiredis-dev ninja-build asciidoc xsltproc docbook-xml docbook-xsl
- name: HTML documentation
os: ubuntu-18.04
EXTRA_CMAKE_BUILD_FLAGS: --target doc-html
RUN_TESTS: none
- apt_get: libzstd-dev asciidoc docbook-xml docbook-xsl
+ apt_get: libzstd-dev pkg-config libhiredis-dev asciidoc docbook-xml docbook-xsl
- name: Manual page
os: ubuntu-18.04
EXTRA_CMAKE_BUILD_FLAGS: --target doc-man-page
RUN_TESTS: none
- apt_get: libzstd-dev asciidoc xsltproc docbook-xml docbook-xsl
+ apt_get: libzstd-dev pkg-config libhiredis-dev asciidoc xsltproc docbook-xml docbook-xsl
- name: Clang-Tidy
os: ubuntu-18.04
CXX: clang++-9
RUN_TESTS: none
CMAKE_PARAMS: -DENABLE_CLANG_TIDY=ON -DCLANGTIDY=/usr/bin/clang-tidy-9
- apt_get: libzstd-dev clang-9 clang-tidy-9
+ apt_get: libzstd-dev pkg-config libhiredis-dev clang-9 clang-tidy-9
steps:
- name: Get source
fetch-depth: 2
- name: Install dependencies
- run: sudo apt-get update && sudo apt-get install ninja-build elfutils libzstd-dev
+ run: sudo apt-get update && sudo apt-get install ninja-build elfutils libzstd-dev pkg-config libhiredis-dev
- name: Initialize CodeQL
uses: github/codeql-action/init@v1
option(ZSTD_FROM_INTERNET "Download and use libzstd from the Internet" ${ZSTD_FROM_INTERNET_DEFAULT})
find_package(zstd 1.1.2 REQUIRED)
+option(REDIS_STORAGE_BACKEND "Enable Redis secondary storage" ON)
+if(REDIS_STORAGE_BACKEND)
+ find_package(hiredis 0.13.3 REQUIRED)
+endif()
+
#
# Special flags
#
--- /dev/null
+find_package(PkgConfig)
+if(PKG_CONFIG_FOUND)
+ pkg_check_modules(HIREDIS REQUIRED hiredis>=${hiredis_FIND_VERSION})
+ find_library(HIREDIS_LIBRARY ${HIREDIS_LIBRARIES} HINTS ${HIREDIS_LIBDIR})
+ find_path(HIREDIS_INCLUDE_DIR hiredis/hiredis.h HINTS ${HIREDIS_PREFIX}/include)
+else()
+ find_library(HIREDIS_LIBRARY hiredis)
+ find_path(HIREDIS_INCLUDE_DIR hiredis/hiredis.h)
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(
+ hiredis "please install libhiredis"
+ HIREDIS_INCLUDE_DIR HIREDIS_LIBRARY)
+mark_as_advanced(HIREDIS_INCLUDE_DIR HIREDIS_LIBRARY)
+
+add_library(HIREDIS::HIREDIS UNKNOWN IMPORTED)
+set_target_properties(
+ HIREDIS::HIREDIS
+ PROPERTIES
+ IMPORTED_LOCATION "${HIREDIS_LIBRARY}"
+ INTERFACE_COMPILE_OPTIONS "${HIREDIS_CFLAGS_OTHER}"
+ INTERFACE_INCLUDE_DIRECTORIES "${HIREDIS_INCLUDE_DIR}")
+
+include(FeatureSummary)
+set_package_properties(
+ hiredis
+ PROPERTIES
+ URL "https://github.com/redis/hiredis"
+ DESCRIPTION "Hiredis is a minimalistic C client library for the Redis database")
* `file:///shared/nfs/directory`
* `file:///shared/nfs/one|read-only file:///shared/nfs/two`
* `http://example.org/cache`
+* `redis://server.example.com`
[[config_sloppiness]] *sloppiness* (*CCACHE_SLOPPINESS*)::
* HTTPS is not yet supported.
* URLs containing IPv6 addresses like `http://[::1]/` are not supported.
+=== Redis storage backend
+
+URL format: `redis://HOST[:PORT]` or `redis://UNIX_SOCKET_PATH`
+
+This backend stores data in a Redis (or Redis-compatible) server.
+There are implementations for both memory and disk based storage.
+
+See <https://redis.io>
+
+Note that ccache will not perform any cleanup of the Redis storage.
+But you can configure LRU eviction: <https://redis.io/topics/lru-cache>
+
+Examples:
+
+* `redis://localhost:6379`
+* `redis:///var/run/redis/redis-server.sock`
+
+Optional attributes:
+
+* *connect-timeout*: Timeout (in ms) for network connection.
+* *operation-timeout*: Timeout (in ms) for Redis commands.
+* *username*: Username for the Redis AUTH command. (Requires ACL)
+* *password*: Password for the Redis AUTH command.
+
== Cache size management
By default, ccache has a 5 GB limit on the total size of files in the cache and
--- /dev/null
+#!/usr/bin/env python3
+
+# this script will upload the contents of the cache,
+# from primary storage to the redis secondary storage
+
+import redis
+import os
+
+config = os.getenv("REDIS_CONF", "localhost")
+if ":" in config:
+ host, port = config.rsplit(":", 1)
+ sock = None
+elif config.startswith("/"):
+ host, port, sock = None, None, config
+else:
+ host, port, sock = config, 6379, None
+username = os.getenv("REDIS_USERNAME")
+password = os.getenv("REDIS_PASSWORD")
+context = redis.Redis(host=host, port=port, unix_socket_path=sock, password=password)
+
+CCACHE_MANIFEST = b"cCmF"
+CCACHE_RESULT = b"cCrS"
+
+ccache = os.getenv("CCACHE_DIR", os.path.expanduser("~/.cache/ccache"))
+filelist = []
+for dirpath, dirnames, filenames in os.walk(ccache):
+ # sort by modification time, most recently used last
+ for filename in filenames:
+ if filename.endswith(".lock"):
+ continue
+ stat = os.stat(os.path.join(dirpath, filename))
+ filelist.append((stat.st_mtime, dirpath, filename))
+filelist.sort()
+files = result = manifest = objects = 0
+for mtime, dirpath, filename in filelist:
+ dirname = dirpath.replace(ccache + os.path.sep, "")
+ if dirname == "tmp":
+ continue
+ elif filename == "CACHEDIR.TAG" or filename == "stats":
+ # ignore these
+ files = files + 1
+ else:
+ (base, ext) = filename[:-1], filename[-1:]
+ if ext == "R" or ext == "M":
+ if ext == "R":
+ result = result + 1
+ if ext == "M":
+ manifest = manifest + 1
+ key = "ccache:" + "".join(list(os.path.split(dirname)) + [base])
+ val = open(os.path.join(dirpath, filename), "rb").read() or None
+ if val:
+ print("%s: %s %d" % (key, ext, len(val)))
+ magic = val[0:4]
+ if ext == "M":
+ assert magic == CCACHE_MANIFEST
+ if ext == "R":
+ assert magic == CCACHE_RESULT
+ context.set(key, val)
+ objects = objects + 1
+ files = files + 1
+print(
+ "%d files, %d result (%d manifest) = %d objects"
+ % (files, result, manifest, objects)
+)
target_include_directories(ccache_lib PRIVATE ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR})
+if(REDIS_STORAGE_BACKEND)
+ target_compile_definitions(ccache_lib PRIVATE -DHAVE_REDIS_STORAGE_BACKEND)
+ target_link_libraries(
+ ccache_lib
+ PUBLIC standard_settings standard_warnings HIREDIS::HIREDIS
+ third_party_lib)
+endif()
+
add_subdirectory(core)
add_subdirectory(storage)
add_subdirectory(third_party)
#include <fmtmacros.hpp>
#include <storage/secondary/FileStorage.hpp>
#include <storage/secondary/HttpStorage.hpp>
+#ifdef HAVE_REDIS_STORAGE_BACKEND
+# include <storage/secondary/RedisStorage.hpp>
+#endif
#include <util/Tokenizer.hpp>
#include <util/string_utils.hpp>
storage_entry.attributes);
}
+#ifdef HAVE_REDIS_STORAGE_BACKEND
+ if (storage_entry.url.scheme() == "redis") {
+ return std::make_unique<secondary::RedisStorage>(storage_entry.url,
+ storage_entry.attributes);
+ }
+#endif
+
return {};
}
${CMAKE_CURRENT_SOURCE_DIR}/HttpStorage.cpp
)
+if(REDIS_STORAGE_BACKEND)
+ list(APPEND sources ${CMAKE_CURRENT_SOURCE_DIR}/RedisStorage.cpp)
+endif()
+
target_sources(ccache_lib PRIVATE ${sources})
--- /dev/null
+// Copyright (C) 2021 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "RedisStorage.hpp"
+
+#include <Digest.hpp>
+#include <Logging.hpp>
+#include <fmtmacros.hpp>
+
+#include <hiredis/hiredis.h>
+
+namespace storage {
+namespace secondary {
+
+const struct timeval DEFAULT_CONNECT_TIMEOUT = {0, 100 * 1000}; // 100 ms
+const struct timeval DEFAULT_OPERATION_TIMEOUT = {10, 0 * 1000}; // 10 sec
+
+static struct timeval
+milliseconds_to_timeval(const std::string& msec)
+{
+ int ms = std::stoi(msec);
+ struct timeval tv;
+ tv.tv_sec = ms / 1000;
+ tv.tv_usec = (ms % 1000) * 1000;
+ return tv;
+}
+
+static std::string
+timeval_to_string(struct timeval tv)
+{
+ return FMT("{:.3f}s", tv.tv_sec + tv.tv_usec / 1000000.0);
+}
+
+static nonstd::optional<struct timeval>
+parse_timeout_attribute(const AttributeMap& attributes, const std::string& name)
+{
+ const auto it = attributes.find(name);
+ if (it == attributes.end()) {
+ return nonstd::nullopt;
+ }
+ return milliseconds_to_timeval(it->second);
+}
+
+static nonstd::optional<std::string>
+parse_string_attribute(const AttributeMap& attributes, const std::string& name)
+{
+ const auto it = attributes.find(name);
+ if (it == attributes.end()) {
+ return nonstd::nullopt;
+ }
+ return it->second;
+}
+
+RedisStorage::RedisStorage(const Url& url, const AttributeMap& attributes)
+ : m_url(url),
+ m_connect_timeout(parse_timeout_attribute(attributes, "connect-timeout")),
+ m_operation_timeout(
+ parse_timeout_attribute(attributes, "operation-timeout")),
+ m_username(parse_string_attribute(attributes, "username")),
+ m_password(parse_string_attribute(attributes, "password"))
+{
+ m_prefix = "ccache"; // TODO: attribute
+ m_context = nullptr;
+ m_connected = false;
+ m_invalid = false;
+}
+
+RedisStorage::~RedisStorage()
+{
+ disconnect();
+ if (m_context) {
+ redisFree(m_context);
+ m_context = nullptr;
+ }
+}
+
+int
+RedisStorage::connect()
+{
+ if (m_connected) {
+ return REDIS_OK;
+ }
+ if (m_invalid) {
+ return REDIS_ERR;
+ }
+
+ if (m_context) {
+ if (redisReconnect(m_context) == REDIS_OK) {
+ m_connected = true;
+ return REDIS_OK;
+ }
+ LOG("Redis reconnect err: {}", m_context->errstr);
+ redisFree(m_context);
+ m_context = nullptr;
+ }
+
+ ASSERT(m_url.scheme() == "redis");
+ std::string host = m_url.host();
+ std::string port = m_url.port();
+ std::string sock = m_url.path();
+ if (m_connect_timeout) {
+ LOG("Redis connect timeout {}", timeval_to_string(*m_connect_timeout));
+ }
+ struct timeval connect_timeout =
+ m_connect_timeout ? *m_connect_timeout : DEFAULT_CONNECT_TIMEOUT;
+ if (!host.empty()) {
+ int p = port.empty() ? 6379 : std::stoi(port);
+ m_context = redisConnectWithTimeout(host.c_str(), p, connect_timeout);
+ } else if (!sock.empty()) {
+ m_context = redisConnectUnixWithTimeout(sock.c_str(), connect_timeout);
+ } else {
+ LOG("Redis invalid url: {}", m_url.str());
+ m_invalid = true;
+ return REDIS_ERR;
+ }
+
+ if (!m_context) {
+ LOG("Redis connect {} err NULL", m_url.str());
+ m_invalid = true;
+ return REDIS_ERR;
+ } else if (m_context->err) {
+ LOG("Redis connect {} err: {}", m_url.str(), m_context->errstr);
+ m_invalid = true;
+ return m_context->err;
+ } else {
+ if (m_context->connection_type == REDIS_CONN_TCP) {
+ LOG(
+ "Redis connect tcp {}:{} OK", m_context->tcp.host, m_context->tcp.port);
+ }
+ if (m_context->connection_type == REDIS_CONN_UNIX) {
+ LOG("Redis connect unix {} OK", m_context->unix_sock.path);
+ }
+ m_connected = true;
+
+ if (m_operation_timeout) {
+ LOG("Redis timeout {}", timeval_to_string(*m_operation_timeout));
+ }
+ struct timeval operation_timeout =
+ m_operation_timeout ? *m_operation_timeout : DEFAULT_OPERATION_TIMEOUT;
+ if (redisSetTimeout(m_context, operation_timeout) != REDIS_OK) {
+ LOG_RAW("Failed to set timeout");
+ }
+
+ return auth();
+ }
+}
+
+int
+RedisStorage::auth()
+{
+ if (m_password) {
+ bool log_password = false;
+ std::string username = m_username ? *m_username : "default";
+ std::string password = log_password ? *m_password : "*******";
+ LOG("Redis AUTH {} {}", username, password);
+ redisReply* reply;
+ if (m_username) {
+ reply = static_cast<redisReply*>(redisCommand(
+ m_context, "AUTH %s %s", m_username->c_str(), m_password->c_str()));
+ } else {
+ reply = static_cast<redisReply*>(
+ redisCommand(m_context, "AUTH %s", m_password->c_str()));
+ }
+ if (!reply) {
+ LOG("Failed to auth {} in redis", username);
+ m_invalid = true;
+ } else if (reply->type == REDIS_REPLY_ERROR) {
+ LOG("Failed to auth {} in redis: {}", username, reply->str);
+ m_invalid = true;
+ }
+ freeReplyObject(reply);
+ if (m_invalid) {
+ return REDIS_ERR;
+ }
+ }
+ return REDIS_OK;
+}
+
+inline bool
+is_error(int err)
+{
+ return (err != REDIS_OK);
+}
+
+inline bool
+is_timeout(int err)
+{
+#ifdef REDIS_ERR_TIMEOUT
+ // Only returned for hiredis version 1.0.0 and above
+ return (err == REDIS_ERR_TIMEOUT);
+#else
+ (void)err;
+ return false;
+#endif
+}
+
+void
+RedisStorage::disconnect()
+{
+ if (m_connected) {
+ // Note: only the async API actually disconnects from the server
+ // the connection is eventually cleaned up in redisFree()
+ LOG_RAW("Redis disconnect");
+ m_connected = false;
+ }
+}
+
+nonstd::expected<nonstd::optional<std::string>, SecondaryStorage::Error>
+RedisStorage::get(const Digest& key)
+{
+ int err = connect();
+ if (is_timeout(err)) {
+ return nonstd::make_unexpected(Error::timeout);
+ } else if (is_error(err)) {
+ return nonstd::make_unexpected(Error::error);
+ }
+ const std::string key_string = get_key_string(key);
+ LOG("Redis GET {}", key_string);
+ redisReply* reply = static_cast<redisReply*>(
+ redisCommand(m_context, "GET %s", key_string.c_str()));
+ bool found = false;
+ bool missing = false;
+ std::string value;
+ if (!reply) {
+ LOG("Failed to get {} from redis", key_string);
+ } else if (reply->type == REDIS_REPLY_ERROR) {
+ LOG("Failed to get {} from redis: {}", key_string, reply->str);
+ } else if (reply->type == REDIS_REPLY_STRING) {
+ value = std::string(reply->str, reply->len);
+ found = true;
+ } else if (reply->type == REDIS_REPLY_NIL) {
+ missing = true;
+ }
+ freeReplyObject(reply);
+ if (found) {
+ return value;
+ } else if (missing) {
+ return nonstd::nullopt;
+ } else {
+ return nonstd::make_unexpected(Error::error);
+ }
+}
+
+nonstd::expected<bool, SecondaryStorage::Error>
+RedisStorage::put(const Digest& key,
+ const std::string& value,
+ bool only_if_missing)
+{
+ int err = connect();
+ if (is_timeout(err)) {
+ return nonstd::make_unexpected(Error::timeout);
+ } else if (is_error(err)) {
+ return nonstd::make_unexpected(Error::error);
+ }
+ const std::string key_string = get_key_string(key);
+ if (only_if_missing) {
+ LOG("Redis EXISTS {}", key_string);
+ redisReply* reply = static_cast<redisReply*>(
+ redisCommand(m_context, "EXISTS %s", key_string.c_str()));
+ int count = 0;
+ if (!reply) {
+ LOG("Failed to check {} in redis", key_string);
+ } else if (reply->type == REDIS_REPLY_ERROR) {
+ LOG("Failed to check {} in redis: {}", key_string, reply->str);
+ } else if (reply->type == REDIS_REPLY_INTEGER) {
+ count = reply->integer;
+ }
+ freeReplyObject(reply);
+ if (count > 0) {
+ return false;
+ }
+ }
+ LOG("Redis SET {}", key_string);
+ redisReply* reply = static_cast<redisReply*>(redisCommand(
+ m_context, "SET %s %b", key_string.c_str(), value.data(), value.size()));
+ bool stored = false;
+ if (!reply) {
+ LOG("Failed to set {} to redis", key_string);
+ } else if (reply->type == REDIS_REPLY_ERROR) {
+ LOG("Failed to set {} to redis: {}", key_string, reply->str);
+ } else if (reply->type == REDIS_REPLY_STATUS) {
+ stored = true;
+ } else {
+ LOG("Failed to set {} to redis: {}", key_string, reply->type);
+ }
+ freeReplyObject(reply);
+ if (stored) {
+ return true;
+ } else {
+ return nonstd::make_unexpected(Error::error);
+ }
+}
+
+nonstd::expected<bool, SecondaryStorage::Error>
+RedisStorage::remove(const Digest& key)
+{
+ int err = connect();
+ if (is_timeout(err)) {
+ return nonstd::make_unexpected(Error::timeout);
+ } else if (is_error(err)) {
+ return nonstd::make_unexpected(Error::error);
+ }
+ const std::string key_string = get_key_string(key);
+ LOG("Redis DEL {}", key_string);
+ redisReply* reply = static_cast<redisReply*>(
+ redisCommand(m_context, "DEL %s", key_string.c_str()));
+ bool removed = false;
+ bool missing = false;
+ if (!reply) {
+ LOG("Failed to del {} in redis", key_string);
+ } else if (reply->type == REDIS_REPLY_ERROR) {
+ LOG("Failed to del {} in redis: {}", key_string, reply->str);
+ } else if (reply->type == REDIS_REPLY_INTEGER) {
+ if (reply->integer > 0) {
+ removed = true;
+ } else {
+ missing = true;
+ }
+ }
+ freeReplyObject(reply);
+ if (removed) {
+ return true;
+ } else if (missing) {
+ return false;
+ } else {
+ return nonstd::make_unexpected(Error::error);
+ }
+}
+
+std::string
+RedisStorage::get_key_string(const Digest& digest) const
+{
+ return FMT("{}:{}", m_prefix, digest.to_string());
+}
+
+} // namespace secondary
+} // namespace storage
--- /dev/null
+// Copyright (C) 2021 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#pragma once
+
+#include "storage/SecondaryStorage.hpp"
+#include "storage/types.hpp"
+
+#include <third_party/url.hpp>
+
+#ifdef HAVE_SYS_TIME_H
+# include <sys/time.h>
+#endif
+
+struct redisContext;
+
+namespace storage {
+namespace secondary {
+
+class RedisStorage : public storage::SecondaryStorage
+{
+public:
+ RedisStorage(const Url& url, const AttributeMap& attributes);
+ ~RedisStorage();
+
+ nonstd::expected<nonstd::optional<std::string>, Error>
+ get(const Digest& key) override;
+ nonstd::expected<bool, Error> put(const Digest& key,
+ const std::string& value,
+ bool only_if_missing) override;
+ nonstd::expected<bool, Error> remove(const Digest& key) override;
+
+private:
+ Url m_url;
+ std::string m_prefix;
+ redisContext* m_context;
+ const nonstd::optional<struct timeval> m_connect_timeout;
+ const nonstd::optional<struct timeval> m_operation_timeout;
+ const nonstd::optional<std::string> m_username;
+ const nonstd::optional<std::string> m_password;
+ bool m_connected;
+ bool m_invalid;
+
+ int connect();
+ int auth();
+ void disconnect();
+ std::string get_key_string(const Digest& digest) const;
+};
+
+} // namespace secondary
+} // namespace storage
addtest(sanitize_blacklist)
addtest(secondary_file)
addtest(secondary_http)
+addtest(secondary_redis)
addtest(secondary_url)
addtest(serialize_diagnostics)
addtest(source_date_epoch)
--- /dev/null
+SUITE_secondary_redis_PROBE() {
+ if ! command -v redis-server &> /dev/null; then
+ echo "redis-server not found"
+ return
+ fi
+ if ! command -v redis-cli &> /dev/null; then
+ echo "redis-cli not found"
+ return
+ fi
+}
+
+SUITE_secondary_redis_SETUP() {
+ unset CCACHE_NODIRECT
+ export CCACHE_SECONDARY_STORAGE="redis://localhost:7777"
+
+ generate_code 1 test.c
+}
+
+expect_number_of_cache_entries() {
+ local expected=$1
+ local url=$2
+ local actual
+
+ actual=$(redis-cli -u "$url" keys "ccache:*" | wc -l)
+ if [ "$actual" -ne "$expected" ]; then
+ test_failed_internal "Found $actual (expected $expected) entries in $url"
+ fi
+}
+
+SUITE_secondary_redis() {
+ if $HOST_OS_APPLE; then
+ # no coreutils on darwin by default, perl rather than gtimeout
+ function timeout() { perl -e 'alarm shift; exec @ARGV' "$@"; }
+ fi
+ timeout 10 redis-server --bind localhost --port 7777 &
+ sleep 0.1 # wait for boot
+ redis-cli -p 7777 ping
+
+ secondary="redis://localhost:7777"
+
+ # -------------------------------------------------------------------------
+ TEST "Base case"
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat 'cache hit (direct)' 0
+ expect_stat 'cache miss' 1
+ expect_stat 'files in cache' 2
+ expect_number_of_cache_entries 2 "$secondary" # result + manifest
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat 'cache hit (direct)' 1
+ expect_stat 'cache miss' 1
+ expect_stat 'files in cache' 2
+ expect_number_of_cache_entries 2 "$secondary" # result + manifest
+
+ $CCACHE -C >/dev/null
+ expect_stat 'files in cache' 0
+ expect_number_of_cache_entries 2 "$secondary" # result + manifest
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat 'cache hit (direct)' 2
+ expect_stat 'cache miss' 1
+ expect_stat 'files in cache' 0
+ expect_number_of_cache_entries 2 "$secondary" # result + manifest
+
+ redis-cli -p 7777 flushdb
+
+ # -------------------------------------------------------------------------
+ TEST "Read-only"
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat 'cache hit (direct)' 0
+ expect_stat 'cache miss' 1
+ expect_stat 'files in cache' 2
+ expect_number_of_cache_entries 2 "$secondary" # result + manifest
+
+ $CCACHE -C >/dev/null
+ expect_stat 'files in cache' 0
+ expect_number_of_cache_entries 2 "$secondary" # result + manifest
+
+ CCACHE_SECONDARY_STORAGE+="|read-only"
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat 'cache hit (direct)' 1
+ expect_stat 'cache miss' 1
+ expect_stat 'files in cache' 0
+ expect_number_of_cache_entries 2 "$secondary" # result + manifest
+
+ echo 'int x;' >> test.c
+ $CCACHE_COMPILE -c test.c
+ expect_stat 'cache hit (direct)' 1
+ expect_stat 'cache miss' 2
+ expect_stat 'files in cache' 2
+ expect_number_of_cache_entries 2 "$secondary" # result + manifest
+}
projects / thirdparty / ccache.git / commitdiff
