Util::setenv("UNCACHED_ERR_FD", FMT("{}", uncached_fd));
}
-static void
-configuration_logger(const std::string& key,
- const std::string& value,
- const std::string& origin)
-{
- BULK_LOG("Config: ({}) {} = {}", origin, key, value);
-}
-
static void
configuration_printer(const std::string& key,
const std::string& value,
}
if (!ctx.config.log_file().empty() || ctx.config.debug()) {
- ctx.config.visit_items(configuration_logger);
+ ctx.config.visit_items([&ctx](const std::string& key,
+ const std::string& value,
+ const std::string& origin) {
+ const auto& log_value =
+ key == "secondary_storage"
+ ? ctx.storage.get_secondary_storage_config_for_logging()
+ : value;
+ BULK_LOG("Config: ({}) {} = {}", origin, key, log_value);
+ });
}
// Guess compiler after logging the config value in order to be able to
+++ /dev/null
-// Copyright (C) 2021 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#pragma once
-
-#include <third_party/nonstd/expected.hpp>
-#include <third_party/nonstd/optional.hpp>
-
-#include <string>
-
-class Digest;
-
-namespace storage {
-
-constexpr auto k_masked_password = "********";
-
-// This class defines the API that a secondary storage backend must implement.
-class SecondaryStorage
-{
-public:
- enum class Error {
- error, // Operation error, e.g. failed connection or authentication.
- timeout, // Timeout, e.g. due to slow network or server.
- };
-
- virtual ~SecondaryStorage() = default;
-
- // Get the value associated with `key`. Returns the value on success or
- // nonstd::nullopt if the entry is not present.
- virtual nonstd::expected<nonstd::optional<std::string>, Error>
- get(const Digest& key) = 0;
-
- // Put `value` associated to `key` in the storage. A true `only_if_missing` is
- // a hint that the value does not have to be set if already present. Returns
- // true if the entry was stored, otherwise false.
- virtual nonstd::expected<bool, Error> put(const Digest& key,
- const std::string& value,
- bool only_if_missing = false) = 0;
-
- // Remove `key` and its associated value. Returns true if the entry was
- // removed, otherwise false.
- virtual nonstd::expected<bool, Error> remove(const Digest& key) = 0;
-};
-
-} // namespace storage
#ifdef HAVE_REDIS_STORAGE_BACKEND
# include <storage/secondary/RedisStorage.hpp>
#endif
+#include <storage/secondary/SecondaryStorage.hpp>
#include <util/Tokenizer.hpp>
+#include <util/expected.hpp>
#include <util/string.hpp>
#include <third_party/url.hpp>
-#include <memory>
-#include <tuple>
+#include <algorithm>
+#include <unordered_map>
namespace storage {
+const std::unordered_map<std::string /*scheme*/,
+ std::shared_ptr<secondary::SecondaryStorage>>
+ k_secondary_storage_implementations = {
+ {"file", std::make_shared<secondary::FileStorage>()},
+ {"http", std::make_shared<secondary::HttpStorage>()},
+#ifdef HAVE_REDIS_STORAGE_BACKEND
+ {"redis", std::make_shared<secondary::RedisStorage>()},
+#endif
+};
+
+struct SecondaryStorageConfig
+{
+ secondary::SecondaryStorage::Backend::Params params;
+ bool read_only = false;
+};
+
+struct SecondaryStorageEntry
+{
+ SecondaryStorageConfig config;
+ std::string url_for_logging;
+ std::shared_ptr<secondary::SecondaryStorage> storage;
+ std::unique_ptr<secondary::SecondaryStorage::Backend> backend;
+ bool failed = false;
+};
+
+static std::string
+to_string(const SecondaryStorageConfig& entry)
+{
+ std::string result = entry.params.url.str();
+ for (const auto& attr : entry.params.attributes) {
+ result += FMT("|{}={}", attr.key, attr.raw_value);
+ }
+ return result;
+}
+
+static SecondaryStorageConfig
+parse_storage_config(const nonstd::string_view entry)
+{
+ const auto parts =
+ Util::split_into_views(entry, "|", util::Tokenizer::Mode::include_empty);
+
+ if (parts.empty() || parts.front().empty()) {
+ throw Error("secondary storage config must provide a URL: {}", entry);
+ }
+
+ SecondaryStorageConfig result;
+ result.params.url = to_string(parts[0]);
+ // The Url class is parsing the URL object lazily; check if successful.
+ try {
+ std::ignore = result.params.url.host();
+ } catch (const Url::parse_error& e) {
+ throw Error("Cannot parse URL: {}", e.what());
+ }
+
+ if (result.params.url.scheme().empty()) {
+ throw Error("URL scheme must not be empty: {}", entry);
+ }
+
+ for (size_t i = 1; i < parts.size(); ++i) {
+ if (parts[i].empty()) {
+ continue;
+ }
+ const auto kv_pair = util::split_once(parts[i], '=');
+ const auto& key = kv_pair.first;
+ const auto& raw_value = kv_pair.second.value_or("true");
+ const auto value =
+ util::value_or_throw<Error>(util::percent_decode(raw_value));
+ if (key == "read-only" && value == "true") {
+ result.read_only = true;
+ }
+ result.params.attributes.emplace_back(
+ secondary::SecondaryStorage::Backend::Attribute{
+ to_string(key), value, to_string(raw_value)});
+ }
+
+ return result;
+}
+
+static std::vector<SecondaryStorageConfig>
+parse_storage_configs(const nonstd::string_view& configs)
+{
+ std::vector<SecondaryStorageConfig> result;
+ for (const auto& config : util::Tokenizer(configs, " ")) {
+ result.push_back(parse_storage_config(config));
+ }
+ return result;
+}
+
+static std::shared_ptr<secondary::SecondaryStorage>
+get_storage(const Url& url)
+{
+ const auto it = k_secondary_storage_implementations.find(url.scheme());
+ if (it != k_secondary_storage_implementations.end()) {
+ return it->second;
+ } else {
+ return {};
+ }
+}
+
Storage::Storage(const Config& config)
: m_config(config),
m_primary_storage(config)
m_primary_storage.finalize();
}
-primary::PrimaryStorage&
-Storage::primary()
-{
- return m_primary_storage;
-}
-
nonstd::optional<std::string>
Storage::get(const Digest& key, const core::CacheEntryType type)
{
return path;
}
- for (const auto& storage : m_secondary_storages) {
- const auto result = storage.backend->get(key);
- if (!result) {
- // The backend is expected to log details about the error.
- // TODO: Update statistics.
- continue;
- }
-
- const auto& value = *result;
- if (!value) {
- LOG("No {} in {}", key.to_string(), storage.url);
- continue;
- }
-
- TemporaryFile tmp_file(FMT("{}/tmp.get", m_config.temporary_dir()));
- m_tmp_files.push_back(tmp_file.path);
- try {
- Util::write_file(tmp_file.path, *value);
- } catch (const Error& e) {
- throw Fatal("Error writing to {}: {}", tmp_file.path, e.what());
- }
-
- LOG("Retrieved {} from {}", key.to_string(), storage.url);
- return tmp_file.path;
+ const auto value = get_from_secondary_storage(key);
+ if (!value) {
+ return nonstd::nullopt;
}
- return nonstd::nullopt;
+ TemporaryFile tmp_file(FMT("{}/tmp.get", m_config.temporary_dir()));
+ m_tmp_files.push_back(tmp_file.path);
+ try {
+ Util::write_file(tmp_file.path, *value);
+ } catch (const Error& e) {
+ throw Fatal("Error writing to {}: {}", tmp_file.path, e.what());
+ }
+ return tmp_file.path;
}
bool
return false;
}
- nonstd::optional<std::string> value;
- for (const auto& storage : m_secondary_storages) {
- if (storage.read_only) {
- LOG("Not storing {} in {} since it is read-only",
- key.to_string(),
- storage.url);
- continue;
- }
-
- if (!value) {
- try {
- value = Util::read_file(*path);
- } catch (const Error& e) {
- LOG("Failed to read {}: {}", *path, e.what());
- break; // Don't indicate failure since primary storage was OK.
- }
- }
-
- const auto result = storage.backend->put(key, *value);
- if (!result) {
- // The backend is expected to log details about the error.
- // TODO: Update statistics.
- continue;
+ // Temporary optimization until primary storage API has been refactored to
+ // pass data via memory instead of files.
+ const bool should_put_in_secondary_storage =
+ std::any_of(m_secondary_storages.begin(),
+ m_secondary_storages.end(),
+ [](const auto& entry) {
+ return !entry->failed && !entry->config.read_only;
+ });
+ if (should_put_in_secondary_storage) {
+ std::string value;
+ try {
+ value = Util::read_file(*path);
+ } catch (const Error& e) {
+ LOG("Failed to read {}: {}", *path, e.what());
+ return true; // Don't indicate failure since primary storage was OK.
}
-
- const bool stored = *result;
- LOG("{} {} in {}",
- stored ? "Stored" : "Failed to store",
- key.to_string(),
- storage.url);
+ put_in_secondary_storage(key, value);
}
return true;
Storage::remove(const Digest& key, const core::CacheEntryType type)
{
m_primary_storage.remove(key, type);
+ remove_from_secondary_storage(key);
+}
- for (const auto& storage : m_secondary_storages) {
- if (storage.read_only) {
- LOG("Did not remove {} from {} since it is read-only",
- key.to_string(),
- storage.url);
- continue;
- }
-
- const auto result = storage.backend->remove(key);
- if (!result) {
- // The backend is expected to log details about the error.
- // TODO: Update statistics.
- continue;
+std::string
+Storage::get_secondary_storage_config_for_logging() const
+{
+ auto configs = parse_storage_configs(m_config.secondary_storage());
+ for (auto& config : configs) {
+ const auto storage = get_storage(config.params.url);
+ if (storage) {
+ storage->redact_secrets(config.params);
}
+ }
+ return util::join(configs, " ");
+}
- const bool removed = *result;
- if (removed) {
- LOG("Removed {} from {}", key.to_string(), storage.url);
- } else {
- LOG("No {} to remove from {}", key.to_string(), storage.url);
+void
+Storage::add_secondary_storages()
+{
+ const auto configs = parse_storage_configs(m_config.secondary_storage());
+ for (const auto& config : configs) {
+ auto url_for_logging = config.params.url;
+ url_for_logging.user_info("");
+ const auto storage = get_storage(config.params.url);
+ if (!storage) {
+ throw Error("unknown secondary storage URL: {}", url_for_logging.str());
}
+ m_secondary_storages.emplace_back(
+ std::make_unique<SecondaryStorageEntry>(SecondaryStorageEntry{
+ config, url_for_logging.str(), storage, {}, false}));
}
}
-namespace {
-
-struct ParseStorageEntryResult
+static void
+mark_backend_as_failed(
+ SecondaryStorageEntry& entry,
+ const secondary::SecondaryStorage::Backend::Failure failure)
{
- Url url;
- storage::AttributeMap attributes;
- bool read_only = false;
-};
-
-} // namespace
+ // The backend is expected to log details about the error.
+ entry.failed = true;
+ (void)failure; // TODO: Update statistics.
+}
-static ParseStorageEntryResult
-parse_storage_entry(const nonstd::string_view& entry)
+static bool
+backend_is_available(SecondaryStorageEntry& entry,
+ nonstd::string_view operation_description,
+ const bool for_writing)
{
- const auto parts =
- Util::split_into_views(entry, "|", util::Tokenizer::Mode::include_empty);
+ if (for_writing && entry.config.read_only) {
+ LOG("Not {} {} since it is read-only",
+ operation_description,
+ entry.url_for_logging);
+ return false;
+ }
- if (parts.empty() || parts.front().empty()) {
- throw Error("secondary storage config must provide a URL: {}", entry);
+ if (entry.failed) {
+ LOG("Not {} {} since it failed earlier",
+ operation_description,
+ entry.url_for_logging);
+ return false;
}
- ParseStorageEntryResult result;
- result.url = std::string(parts[0]);
- // Url class is parsing the URL object lazily; check if successful
try {
- std::ignore = result.url.host();
- } catch (Url::parse_error& e) {
- throw Error("Cannot parse URL: {}", e.what());
+ entry.backend = entry.storage->create_backend(entry.config.params);
+ } catch (const secondary::SecondaryStorage::Backend::Failed& e) {
+ LOG("Failed to construct backend for {}{}",
+ entry.url_for_logging,
+ nonstd::string_view(e.what()).empty() ? "" : FMT(": {}", e.what()));
+ mark_backend_as_failed(entry, e.failure());
+ return false;
}
- if (result.url.scheme().empty()) {
- throw Error("URL scheme must not be empty: {}", entry);
- }
+ return true;
+}
- for (size_t i = 1; i < parts.size(); ++i) {
- if (parts[i].empty()) {
+nonstd::optional<std::string>
+Storage::get_from_secondary_storage(const Digest& key)
+{
+ for (const auto& entry : m_secondary_storages) {
+ if (!backend_is_available(*entry, "getting from", false)) {
continue;
}
- const auto kv_pair = util::split_once(parts[i], '=');
- const auto& key = kv_pair.first;
- const auto& value = kv_pair.second ? *kv_pair.second : "true";
- const auto decoded_value = util::percent_decode(value);
- if (!decoded_value) {
- throw Error(decoded_value.error());
+
+ const auto result = entry->backend->get(key);
+ if (!result) {
+ mark_backend_as_failed(*entry, result.error());
+ continue;
}
- if (key == "read-only" && value == "true") {
- result.read_only = true;
+
+ const auto& value = *result;
+ if (value) {
+ LOG("Retrieved {} from {}", key.to_string(), entry->url_for_logging);
+ return *value;
} else {
- result.attributes.emplace(std::string(key), *decoded_value);
+ LOG("No {} in {}", key.to_string(), entry->url_for_logging);
}
}
- return result;
+ return nonstd::nullopt;
}
-static std::unique_ptr<SecondaryStorage>
-create_storage(const ParseStorageEntryResult& storage_entry)
+void
+Storage::put_in_secondary_storage(const Digest& key, const std::string& value)
{
- if (storage_entry.url.scheme() == "file") {
- return std::make_unique<secondary::FileStorage>(storage_entry.url,
- storage_entry.attributes);
- } else if (storage_entry.url.scheme() == "http") {
- return std::make_unique<secondary::HttpStorage>(storage_entry.url,
- storage_entry.attributes);
- }
+ for (const auto& entry : m_secondary_storages) {
+ if (!backend_is_available(*entry, "putting in", true)) {
+ continue;
+ }
-#ifdef HAVE_REDIS_STORAGE_BACKEND
- if (storage_entry.url.scheme() == "redis") {
- return std::make_unique<secondary::RedisStorage>(storage_entry.url,
- storage_entry.attributes);
- }
-#endif
+ const auto result = entry->backend->put(key, value);
+ if (!result) {
+ // The backend is expected to log details about the error.
+ mark_backend_as_failed(*entry, result.error());
+ continue;
+ }
- return {};
+ const bool stored = *result;
+ LOG("{} {} in {}",
+ stored ? "Stored" : "Failed to store",
+ key.to_string(),
+ entry->url_for_logging);
+ }
}
void
-Storage::add_secondary_storages()
+Storage::remove_from_secondary_storage(const Digest& key)
{
- for (const auto& entry : util::Tokenizer(m_config.secondary_storage(), " ")) {
- const auto storage_entry = parse_storage_entry(entry);
- auto storage = create_storage(storage_entry);
- auto url_for_logging = storage_entry.url.str();
- if (!storage) {
- throw Error("unknown secondary storage URL: {}", url_for_logging);
+ for (const auto& entry : m_secondary_storages) {
+ if (!backend_is_available(*entry, "removing from", true)) {
+ continue;
+ }
+
+ const auto result = entry->backend->remove(key);
+ if (!result) {
+ mark_backend_as_failed(*entry, result.error());
+ continue;
+ }
+
+ const bool removed = *result;
+ if (removed) {
+ LOG("Removed {} from {}", key.to_string(), entry->url_for_logging);
+ } else {
+ LOG("No {} to remove from {}", key.to_string(), entry->url_for_logging);
}
- m_secondary_storages.push_back(SecondaryStorageEntry{
- std::move(storage), url_for_logging, storage_entry.read_only});
}
}
#include "types.hpp"
#include <core/types.hpp>
-#include <storage/SecondaryStorage.hpp>
#include <storage/primary/PrimaryStorage.hpp>
#include <third_party/nonstd/optional.hpp>
#include <functional>
+#include <memory>
#include <string>
#include <vector>
namespace storage {
+struct SecondaryStorageEntry;
+
class Storage
{
public:
void remove(const Digest& key, core::CacheEntryType type);
-private:
- struct SecondaryStorageEntry
- {
- std::unique_ptr<storage::SecondaryStorage> backend;
- std::string url; // the Url class has a too-chatty stream operator
- bool read_only = false;
- };
+ std::string get_secondary_storage_config_for_logging() const;
+private:
const Config& m_config;
primary::PrimaryStorage m_primary_storage;
- std::vector<SecondaryStorageEntry> m_secondary_storages;
+ std::vector<std::unique_ptr<SecondaryStorageEntry>> m_secondary_storages;
std::vector<std::string> m_tmp_files;
void add_secondary_storages();
+ nonstd::optional<std::string> get_from_secondary_storage(const Digest& key);
+ void put_in_secondary_storage(const Digest& key, const std::string& value);
+ void remove_from_secondary_storage(const Digest& key);
};
+inline primary::PrimaryStorage&
+Storage::primary()
+{
+ return m_primary_storage;
+}
+
} // namespace storage
sources
${CMAKE_CURRENT_SOURCE_DIR}/FileStorage.cpp
${CMAKE_CURRENT_SOURCE_DIR}/HttpStorage.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/SecondaryStorage.cpp
)
if(REDIS_STORAGE_BACKEND)
#include <UmaskScope.hpp>
#include <Util.hpp>
#include <assertions.hpp>
+#include <exceptions.hpp>
#include <fmtmacros.hpp>
+#include <util/expected.hpp>
#include <util/file.hpp>
#include <util/string.hpp>
#include <third_party/nonstd/string_view.hpp>
+#include <sys/stat.h> // for mode_t
+
namespace storage {
namespace secondary {
-static std::string
-parse_url(const Url& url)
-{
- ASSERT(url.scheme() == "file");
- const auto& dir = url.path();
- if (!util::starts_with(dir, "/")) {
- throw Error("invalid file path \"{}\" - directory must start with a slash",
- dir);
- }
- return dir;
-}
+namespace {
-static nonstd::optional<mode_t>
-parse_umask(const AttributeMap& attributes)
+class FileStorageBackend : public SecondaryStorage::Backend
{
- const auto it = attributes.find("umask");
- if (it == attributes.end()) {
- return nonstd::nullopt;
- }
+public:
+ FileStorageBackend(const Params& params);
- const auto umask = util::parse_umask(it->second);
- if (umask) {
- return *umask;
- } else {
- LOG("Error: {}", umask.error());
- return nonstd::nullopt;
- }
-}
+ nonstd::expected<nonstd::optional<std::string>, Failure>
+ get(const Digest& key) override;
-static bool
-parse_update_mtime(const AttributeMap& attributes)
-{
- const auto it = attributes.find("update-mtime");
- return it != attributes.end() && it->second == "true";
-}
+ nonstd::expected<bool, Failure> put(const Digest& key,
+ const std::string& value,
+ bool only_if_missing) override;
-FileStorage::FileStorage(const Url& url, const AttributeMap& attributes)
- : m_dir(parse_url(url)),
- m_umask(parse_umask(attributes)),
- m_update_mtime(parse_update_mtime(attributes))
+ nonstd::expected<bool, Failure> remove(const Digest& key) override;
+
+private:
+ const std::string m_dir;
+ nonstd::optional<mode_t> m_umask;
+ bool m_update_mtime = false;
+
+ std::string get_entry_path(const Digest& key) const;
+};
+
+FileStorageBackend::FileStorageBackend(const Params& params)
+ : m_dir(params.url.path())
{
+ ASSERT(params.url.scheme() == "file");
+ if (!params.url.host().empty()) {
+ throw Fatal(FMT(
+ "invalid file path \"{}\": specifying a host (\"{}\") is not supported",
+ params.url.str(),
+ params.url.host()));
+ }
+
+ for (const auto& attr : params.attributes) {
+ if (attr.key == "umask") {
+ m_umask = util::value_or_throw<Fatal>(util::parse_umask(attr.value));
+ } else if (attr.key == "update-mtime") {
+ m_update_mtime = attr.value == "true";
+ } else if (!is_framework_attribute(attr.key)) {
+ LOG("Unknown attribute: {}", attr.key);
+ }
+ }
}
-nonstd::expected<nonstd::optional<std::string>, SecondaryStorage::Error>
-FileStorage::get(const Digest& key)
+nonstd::expected<nonstd::optional<std::string>,
+ SecondaryStorage::Backend::Failure>
+FileStorageBackend::get(const Digest& key)
{
const auto path = get_entry_path(key);
const bool exists = Stat::stat(path);
try {
LOG("Reading {}", path);
return Util::read_file(path);
- } catch (const ::Error& e) {
+ } catch (const Error& e) {
LOG("Failed to read {}: {}", path, e.what());
- return nonstd::make_unexpected(Error::error);
+ return nonstd::make_unexpected(Failure::error);
}
}
-nonstd::expected<bool, SecondaryStorage::Error>
-FileStorage::put(const Digest& key,
- const std::string& value,
- const bool only_if_missing)
+nonstd::expected<bool, SecondaryStorage::Backend::Failure>
+FileStorageBackend::put(const Digest& key,
+ const std::string& value,
+ const bool only_if_missing)
{
const auto path = get_entry_path(key);
const auto dir = Util::dir_name(path);
if (!Util::create_dir(dir)) {
LOG("Failed to create directory {}: {}", dir, strerror(errno));
- return nonstd::make_unexpected(Error::error);
+ return nonstd::make_unexpected(Failure::error);
}
LOG("Writing {}", path);
file.write(value);
file.commit();
return true;
- } catch (const ::Error& e) {
+ } catch (const Error& e) {
LOG("Failed to write {}: {}", path, e.what());
- return nonstd::make_unexpected(Error::error);
+ return nonstd::make_unexpected(Failure::error);
}
}
}
-nonstd::expected<bool, SecondaryStorage::Error>
-FileStorage::remove(const Digest& key)
+nonstd::expected<bool, SecondaryStorage::Backend::Failure>
+FileStorageBackend::remove(const Digest& key)
{
return Util::unlink_safe(get_entry_path(key));
}
std::string
-FileStorage::get_entry_path(const Digest& key) const
+FileStorageBackend::get_entry_path(const Digest& key) const
{
const auto key_string = key.to_string();
const uint8_t digits = 2;
return FMT("{}/{:.{}}/{}", m_dir, key_string, digits, &key_string[digits]);
}
+} // namespace
+
+std::unique_ptr<SecondaryStorage::Backend>
+FileStorage::create_backend(const Backend::Params& params) const
+{
+ return std::make_unique<FileStorageBackend>(params);
+}
+
} // namespace secondary
} // namespace storage
#pragma once
-#include <storage/SecondaryStorage.hpp>
-#include <storage/types.hpp>
-
-#include <third_party/url.hpp>
-
-#include <sys/stat.h> // for mode_t
+#include "SecondaryStorage.hpp"
namespace storage {
namespace secondary {
class FileStorage : public SecondaryStorage
{
public:
- FileStorage(const Url& url, const AttributeMap& attributes);
-
- nonstd::expected<nonstd::optional<std::string>, Error>
- get(const Digest& key) override;
- nonstd::expected<bool, Error> put(const Digest& key,
- const std::string& value,
- bool only_if_missing) override;
- nonstd::expected<bool, Error> remove(const Digest& key) override;
-
-private:
- const std::string m_dir;
- const nonstd::optional<mode_t> m_umask;
- const bool m_update_mtime;
-
- std::string get_entry_path(const Digest& key) const;
+ std::unique_ptr<Backend>
+ create_backend(const Backend::Params& params) const override;
};
} // namespace secondary
namespace {
-const auto DEFAULT_CONNECT_TIMEOUT = std::chrono::milliseconds{100};
-const auto DEFAULT_OPERATION_TIMEOUT = std::chrono::milliseconds{10000};
+class HttpStorageBackend : public SecondaryStorage::Backend
+{
+public:
+ HttpStorageBackend(const Params& params);
+
+ nonstd::expected<nonstd::optional<std::string>, Failure>
+ get(const Digest& key) override;
+
+ nonstd::expected<bool, Failure> put(const Digest& key,
+ const std::string& value,
+ bool only_if_missing) override;
+
+ nonstd::expected<bool, Failure> remove(const Digest& key) override;
+
+private:
+ const std::string m_url_path;
+ httplib::Client m_http_client;
+
+ std::string get_entry_path(const Digest& key) const;
+};
nonstd::string_view
to_string(const httplib::Error error)
const auto rendered_value = host_and_port_only.str();
const auto prefix = nonstd::string_view{"//"};
if (!util::starts_with(rendered_value, prefix)) {
- throw Error(
- "Expected partial URL to start with '{}': '{}'", prefix, rendered_value);
+ throw Fatal(R"(Expected partial URL "{}" to start with "{}")",
+ rendered_value,
+ prefix);
}
return rendered_value.substr(prefix.size());
}
-std::unique_ptr<httplib::Client>
-make_client(const Url& url)
+std::string
+get_url(const Url& url)
{
if (url.host().empty()) {
- throw Error("A host is required in HTTP storage URL: '{}'", url.str());
+ throw Fatal("A host is required in HTTP storage URL \"{}\"", url.str());
}
- // the httplib requires a partial URL with just scheme, host and port
+ // httplib requires a partial URL with just scheme, host and port.
Url destination;
destination.scheme(url.scheme())
.host(url.host(), url.ip_version())
.port(url.port());
+ return destination.str();
+}
- auto client = std::make_unique<httplib::Client>(destination.str().c_str());
- if (!url.user_info().empty()) {
- const auto pair = util::split_once(url.user_info(), ':');
+HttpStorageBackend::HttpStorageBackend(const Params& params)
+ : m_url_path(get_url_path(params.url)),
+ m_http_client(get_url(params.url).c_str())
+{
+ if (!params.url.user_info().empty()) {
+ const auto pair = util::split_once(params.url.user_info(), ':');
if (!pair.second) {
- throw Error("Expected username:password in URL but got: '{}'",
- url.user_info());
+ throw Fatal("Expected username:password in URL but got \"{}\"",
+ params.url.user_info());
}
- client->set_basic_auth(nonstd::sv_lite::to_string(pair.first).c_str(),
- nonstd::sv_lite::to_string(*pair.second).c_str());
+ m_http_client.set_basic_auth(to_string(pair.first).c_str(),
+ to_string(*pair.second).c_str());
}
- return client;
-}
-
-std::chrono::milliseconds
-parse_timeout_attribute(const AttributeMap& attributes,
- const std::string& name,
- const std::chrono::milliseconds default_value)
-{
- const auto it = attributes.find(name);
- if (it == attributes.end()) {
- return default_value;
- } else {
- const auto timeout_in_ms = util::value_or_throw<Error>(
- util::parse_unsigned(it->second, 1, 1000 * 3600, "timeout"));
- return std::chrono::milliseconds{timeout_in_ms};
+ m_http_client.set_default_headers({
+ // Explicit setting of the Host header is required due to IPv6 address
+ // handling issues in httplib.
+ {"Host", get_host_header_value(params.url)},
+ {"User-Agent", FMT("{}/{}", CCACHE_NAME, CCACHE_VERSION)},
+ });
+ m_http_client.set_keep_alive(true);
+
+ auto connect_timeout = k_default_connect_timeout;
+ auto operation_timeout = k_default_operation_timeout;
+
+ for (const auto& attr : params.attributes) {
+ if (attr.key == "connect-timeout") {
+ connect_timeout = parse_timeout_attribute(attr.value);
+ } else if (attr.key == "operation-timeout") {
+ operation_timeout = parse_timeout_attribute(attr.value);
+ } else if (!is_framework_attribute(attr.key)) {
+ LOG("Unknown attribute: {}", attr.key);
+ }
}
-}
-
-} // namespace
-HttpStorage::HttpStorage(const Url& url, const AttributeMap& attributes)
- : m_url_path(get_url_path(url)),
- m_http_client(make_client(url))
-{
- m_http_client->set_default_headers(
- {// explicit setting of the Host header is required due to IPv6 address
- // handling issues in httplib
- {"Host", get_host_header_value(url)},
- {"User-Agent", FMT("{}/{}", CCACHE_NAME, CCACHE_VERSION)}});
- m_http_client->set_keep_alive(true);
- configure_timeouts(attributes);
-}
-
-HttpStorage::~HttpStorage() = default;
-
-void
-HttpStorage::configure_timeouts(const AttributeMap& attributes)
-{
- const auto connection_timeout = parse_timeout_attribute(
- attributes, "connect-timeout", DEFAULT_CONNECT_TIMEOUT);
- const auto operation_timeout = parse_timeout_attribute(
- attributes, "operation-timeout", DEFAULT_OPERATION_TIMEOUT);
-
- m_http_client->set_connection_timeout(connection_timeout);
- m_http_client->set_read_timeout(operation_timeout);
- m_http_client->set_write_timeout(operation_timeout);
+ m_http_client.set_connection_timeout(connect_timeout);
+ m_http_client.set_read_timeout(operation_timeout);
+ m_http_client.set_write_timeout(operation_timeout);
}
-nonstd::expected<nonstd::optional<std::string>, SecondaryStorage::Error>
-HttpStorage::get(const Digest& key)
+nonstd::expected<nonstd::optional<std::string>,
+ SecondaryStorage::Backend::Failure>
+HttpStorageBackend::get(const Digest& key)
{
const auto url_path = get_entry_path(key);
- const auto result = m_http_client->Get(url_path.c_str());
+ const auto result = m_http_client.Get(url_path.c_str());
if (result.error() != httplib::Error::Success || !result) {
LOG("Failed to get {} from http storage: {} ({})",
url_path,
to_string(result.error()),
result.error());
- return nonstd::make_unexpected(Error::error);
+ return nonstd::make_unexpected(Failure::error);
}
if (result->status < 200 || result->status >= 300) {
return result->body;
}
-nonstd::expected<bool, SecondaryStorage::Error>
-HttpStorage::put(const Digest& key,
- const std::string& value,
- const bool only_if_missing)
+nonstd::expected<bool, SecondaryStorage::Backend::Failure>
+HttpStorageBackend::put(const Digest& key,
+ const std::string& value,
+ const bool only_if_missing)
{
const auto url_path = get_entry_path(key);
if (only_if_missing) {
- const auto result = m_http_client->Head(url_path.c_str());
+ const auto result = m_http_client.Head(url_path.c_str());
if (result.error() != httplib::Error::Success || !result) {
LOG("Failed to check for {} in http storage: {} ({})",
url_path,
to_string(result.error()),
result.error());
- return nonstd::make_unexpected(Error::error);
+ return nonstd::make_unexpected(Failure::error);
}
if (result->status >= 200 && result->status < 300) {
}
static const auto content_type = "application/octet-stream";
- const auto result = m_http_client->Put(
+ const auto result = m_http_client.Put(
url_path.c_str(), value.data(), value.size(), content_type);
if (result.error() != httplib::Error::Success || !result) {
url_path,
to_string(result.error()),
result.error());
- return nonstd::make_unexpected(Error::error);
+ return nonstd::make_unexpected(Failure::error);
}
if (result->status < 200 || result->status >= 300) {
LOG("Failed to put {} to http storage: status code: {}",
url_path,
result->status);
- return nonstd::make_unexpected(Error::error);
+ return nonstd::make_unexpected(Failure::error);
}
return true;
}
-nonstd::expected<bool, SecondaryStorage::Error>
-HttpStorage::remove(const Digest& key)
+nonstd::expected<bool, SecondaryStorage::Backend::Failure>
+HttpStorageBackend::remove(const Digest& key)
{
const auto url_path = get_entry_path(key);
- const auto result = m_http_client->Delete(url_path.c_str());
+ const auto result = m_http_client.Delete(url_path.c_str());
if (result.error() != httplib::Error::Success || !result) {
LOG("Failed to delete {} from http storage: {} ({})",
url_path,
to_string(result.error()),
result.error());
- return nonstd::make_unexpected(Error::error);
+ return nonstd::make_unexpected(Failure::error);
}
if (result->status < 200 || result->status >= 300) {
LOG("Failed to delete {} from http storage: status code: {}",
url_path,
result->status);
- return nonstd::make_unexpected(Error::error);
+ return nonstd::make_unexpected(Failure::error);
}
return true;
}
std::string
-HttpStorage::get_entry_path(const Digest& key) const
+HttpStorageBackend::get_entry_path(const Digest& key) const
{
return m_url_path + key.to_string();
}
+} // namespace
+
+std::unique_ptr<SecondaryStorage::Backend>
+HttpStorage::create_backend(const Backend::Params& params) const
+{
+ return std::make_unique<HttpStorageBackend>(params);
+}
+
+void
+HttpStorage::redact_secrets(Backend::Params& params) const
+{
+ auto& url = params.url;
+ const auto user_info = util::split_once(url.user_info(), ':');
+ if (user_info.second) {
+ url.user_info(FMT("{}:{}", user_info.first, k_redacted_password));
+ }
+}
+
} // namespace secondary
} // namespace storage
#pragma once
-#include <storage/SecondaryStorage.hpp>
-#include <storage/types.hpp>
-
-#include <memory>
-#include <string>
-
-class Url;
-
-namespace httplib {
-class Client;
-}
+#include "SecondaryStorage.hpp"
namespace storage {
namespace secondary {
class HttpStorage : public SecondaryStorage
{
public:
- HttpStorage(const Url& url, const AttributeMap& attributes);
- ~HttpStorage() override;
-
- nonstd::expected<nonstd::optional<std::string>, Error>
- get(const Digest& key) override;
- nonstd::expected<bool, Error> put(const Digest& key,
- const std::string& value,
- bool only_if_missing) override;
- nonstd::expected<bool, Error> remove(const Digest& key) override;
-
-private:
- const std::string m_url_path;
- std::unique_ptr<httplib::Client> m_http_client;
+ std::unique_ptr<Backend>
+ create_backend(const Backend::Params& params) const override;
- void configure_timeouts(const AttributeMap& attributes);
- std::string get_entry_path(const Digest& key) const;
+ void redact_secrets(Backend::Params& params) const override;
};
} // namespace secondary
#include <Digest.hpp>
#include <Logging.hpp>
+#include <exceptions.hpp>
#include <fmtmacros.hpp>
#include <util/expected.hpp>
#include <util/string.hpp>
namespace storage {
namespace secondary {
-const uint64_t DEFAULT_CONNECT_TIMEOUT_MS = 100;
-const uint64_t DEFAULT_OPERATION_TIMEOUT_MS = 10000;
-const uint32_t DEFAULT_PORT = 6379;
+namespace {
+using RedisContext = std::unique_ptr<redisContext, decltype(&redisFree)>;
using RedisReply = std::unique_ptr<redisReply, decltype(&freeReplyObject)>;
-static RedisReply
-redis_command(redisContext* context, const char* format, ...)
+const uint32_t DEFAULT_PORT = 6379;
+
+class RedisStorageBackend : public SecondaryStorage::Backend
{
- va_list ap;
- va_start(ap, format);
- void* reply = redisvCommand(context, format, ap);
- va_end(ap);
- return RedisReply(static_cast<redisReply*>(reply), freeReplyObject);
-}
+public:
+ RedisStorageBackend(const SecondaryStorage::Backend::Params& params);
+
+ nonstd::expected<nonstd::optional<std::string>, Failure>
+ get(const Digest& key) override;
-static struct timeval
-milliseconds_to_timeval(const uint64_t ms)
+ nonstd::expected<bool, Failure> put(const Digest& key,
+ const std::string& value,
+ bool only_if_missing) override;
+
+ nonstd::expected<bool, Failure> remove(const Digest& key) override;
+
+private:
+ const std::string m_prefix;
+ RedisContext m_context;
+
+ void
+ connect(const Url& url, uint32_t connect_timeout, uint32_t operation_timeout);
+ void select_database(const Url& url);
+ void authenticate(const Url& url);
+ nonstd::expected<RedisReply, Failure> redis_command(const char* format, ...);
+ std::string get_key_string(const Digest& digest) const;
+};
+
+timeval
+to_timeval(const uint32_t ms)
{
- struct timeval tv;
+ timeval tv;
tv.tv_sec = ms / 1000;
tv.tv_usec = (ms % 1000) * 1000;
return tv;
}
-static uint64_t
-parse_timeout_attribute(const AttributeMap& attributes,
- const std::string& name,
- const uint64_t default_value)
-{
- const auto it = attributes.find(name);
- if (it == attributes.end()) {
- return default_value;
- } else {
- return util::value_or_throw<Error>(
- util::parse_unsigned(it->second, 1, 1000 * 3600, "timeout"));
- }
-}
-
-static std::pair<nonstd::optional<std::string>, nonstd::optional<std::string>>
+std::pair<nonstd::optional<std::string>, nonstd::optional<std::string>>
split_user_info(const std::string& user_info)
{
const auto pair = util::split_once(user_info, ':');
}
}
-RedisStorage::RedisStorage(const Url& url, const AttributeMap& attributes)
- : m_url(url),
- m_prefix("ccache"), // TODO: attribute
- m_context(nullptr),
- m_connect_timeout(parse_timeout_attribute(
- attributes, "connect-timeout", DEFAULT_CONNECT_TIMEOUT_MS)),
- m_operation_timeout(parse_timeout_attribute(
- attributes, "operation-timeout", DEFAULT_OPERATION_TIMEOUT_MS)),
- m_connected(false),
- m_invalid(false)
-{
-}
-
-RedisStorage::~RedisStorage()
+RedisStorageBackend::RedisStorageBackend(const Params& params)
+ : m_prefix("ccache"), // TODO: attribute
+ m_context(nullptr, redisFree)
{
- if (m_context) {
- LOG_RAW("Redis disconnect");
- redisFree(m_context);
- m_context = nullptr;
- }
-}
-
-int
-RedisStorage::connect()
-{
- if (m_connected) {
- return REDIS_OK;
- }
- if (m_invalid) {
- return REDIS_ERR;
- }
-
- if (m_context) {
- if (redisReconnect(m_context) == REDIS_OK) {
- m_connected = true;
- return REDIS_OK;
- }
- LOG("Redis reconnection error: {}", m_context->errstr);
- redisFree(m_context);
- m_context = nullptr;
- }
-
- ASSERT(m_url.scheme() == "redis");
- const std::string host = m_url.host().empty() ? "localhost" : m_url.host();
- const uint32_t port = m_url.port().empty()
- ? DEFAULT_PORT
- : util::value_or_throw<::Error>(util::parse_unsigned(
- m_url.port(), 1, 65535, "port"));
- ASSERT(m_url.path().empty() || m_url.path()[0] == '/');
- const uint32_t db_number =
- m_url.path().empty() ? 0
- : util::value_or_throw<::Error>(util::parse_unsigned(
- m_url.path().substr(1),
- 0,
- std::numeric_limits<uint32_t>::max(),
- "db number"));
-
- const auto connect_timeout = milliseconds_to_timeval(m_connect_timeout);
-
- LOG("Redis connecting to {}:{} (timeout {} ms)",
- host.c_str(),
- port,
- m_connect_timeout);
- m_context = redisConnectWithTimeout(host.c_str(), port, connect_timeout);
-
- if (!m_context) {
- LOG_RAW("Redis connection error (NULL context)");
- m_invalid = true;
- return REDIS_ERR;
- } else if (m_context->err) {
- LOG("Redis connection error: {}", m_context->errstr);
- m_invalid = true;
- return m_context->err;
- }
-
- LOG("Redis connection to {}:{} OK", m_context->tcp.host, m_context->tcp.port);
- m_connected = true;
-
- if (redisSetTimeout(m_context, milliseconds_to_timeval(m_operation_timeout))
- != REDIS_OK) {
- LOG_RAW("Failed to set operation timeout");
- }
-
- if (db_number != 0) {
- LOG("Redis SELECT {}", db_number);
- const auto reply = redis_command(m_context, "SELECT %d", db_number);
- if (!reply) {
- LOG_RAW("Redis SELECT failed (NULL)");
- m_invalid = true;
- return REDIS_ERR;
- } else if (reply->type == REDIS_REPLY_ERROR) {
- LOG("Redis SELECT error: {}", reply->str);
- m_invalid = true;
- return REDIS_ERR;
- }
- }
-
- return auth();
-}
-
-int
-RedisStorage::auth()
-{
- const auto password_username_pair = split_user_info(m_url.user_info());
- const auto& password = password_username_pair.first;
- if (password) {
- RedisReply reply(nullptr, freeReplyObject);
- const auto& username = password_username_pair.second;
- if (username) {
- LOG("Redis AUTH {} {}", *username, storage::k_masked_password);
- reply = redis_command(
- m_context, "AUTH %s %s", username->c_str(), password->c_str());
- } else {
- LOG("Redis AUTH {}", storage::k_masked_password);
- reply = redis_command(m_context, "AUTH %s", password->c_str());
- }
- if (!reply) {
- LOG_RAW("Redis AUTH failed (NULL)");
- m_invalid = true;
- return REDIS_ERR;
- } else if (reply->type == REDIS_REPLY_ERROR) {
- LOG("Redis AUTH error: {}", reply->str);
- m_invalid = true;
- return REDIS_ERR;
+ const auto& url = params.url;
+ ASSERT(url.scheme() == "redis");
+
+ auto connect_timeout = k_default_connect_timeout;
+ auto operation_timeout = k_default_operation_timeout;
+
+ for (const auto& attr : params.attributes) {
+ if (attr.key == "connect-timeout") {
+ connect_timeout = parse_timeout_attribute(attr.value);
+ } else if (attr.key == "operation-timeout") {
+ operation_timeout = parse_timeout_attribute(attr.value);
+ } else if (!is_framework_attribute(attr.key)) {
+ LOG("Unknown attribute: {}", attr.key);
}
}
- return REDIS_OK;
+ connect(url, connect_timeout.count(), operation_timeout.count());
+ select_database(url);
+ authenticate(url);
}
inline bool
#endif
}
-nonstd::expected<nonstd::optional<std::string>, SecondaryStorage::Error>
-RedisStorage::get(const Digest& key)
+nonstd::expected<nonstd::optional<std::string>,
+ SecondaryStorage::Backend::Failure>
+RedisStorageBackend::get(const Digest& key)
{
- const int err = connect();
- if (is_timeout(err)) {
- return nonstd::make_unexpected(Error::timeout);
- } else if (is_error(err)) {
- return nonstd::make_unexpected(Error::error);
- }
-
- const std::string key_string = get_key_string(key);
+ const auto key_string = get_key_string(key);
LOG("Redis GET {}", key_string);
-
- const auto reply = redis_command(m_context, "GET %s", key_string.c_str());
+ const auto reply = redis_command("GET %s", key_string.c_str());
if (!reply) {
- LOG("Failed to get {} from Redis (NULL)", key_string);
- } else if (reply->type == REDIS_REPLY_STRING) {
- return std::string(reply->str, reply->len);
- } else if (reply->type == REDIS_REPLY_NIL) {
+ return nonstd::make_unexpected(reply.error());
+ } else if ((*reply)->type == REDIS_REPLY_STRING) {
+ return std::string((*reply)->str, (*reply)->len);
+ } else if ((*reply)->type == REDIS_REPLY_NIL) {
return nonstd::nullopt;
- } else if (reply->type == REDIS_REPLY_ERROR) {
- LOG("Failed to get {} from Redis: {}", key_string, reply->str);
} else {
- LOG("Failed to get {} from Redis: unknown reply type {}",
- key_string,
- reply->type);
+ LOG("Unknown reply type: {}", (*reply)->type);
+ return nonstd::make_unexpected(Failure::error);
}
-
- return nonstd::make_unexpected(Error::error);
}
-nonstd::expected<bool, SecondaryStorage::Error>
-RedisStorage::put(const Digest& key,
- const std::string& value,
- bool only_if_missing)
+nonstd::expected<bool, SecondaryStorage::Backend::Failure>
+RedisStorageBackend::put(const Digest& key,
+ const std::string& value,
+ bool only_if_missing)
{
- const int err = connect();
- if (is_timeout(err)) {
- return nonstd::make_unexpected(Error::timeout);
- } else if (is_error(err)) {
- return nonstd::make_unexpected(Error::error);
- }
+ const auto key_string = get_key_string(key);
- const std::string key_string = get_key_string(key);
if (only_if_missing) {
LOG("Redis EXISTS {}", key_string);
- const auto reply =
- redis_command(m_context, "EXISTS %s", key_string.c_str());
+ const auto reply = redis_command("EXISTS %s", key_string.c_str());
if (!reply) {
- LOG("Failed to check {} in Redis", key_string);
- } else if (reply->type == REDIS_REPLY_INTEGER && reply->integer > 0) {
+ return nonstd::make_unexpected(reply.error());
+ } else if ((*reply)->type == REDIS_REPLY_INTEGER && (*reply)->integer > 0) {
LOG("Entry {} already in Redis", key_string);
return false;
- } else if (reply->type == REDIS_REPLY_ERROR) {
- LOG("Failed to check {} in Redis: {}", key_string, reply->str);
+ } else {
+ LOG("Unknown reply type: {}", (*reply)->type);
}
}
- LOG("Redis SET {}", key_string);
- const auto reply = redis_command(
- m_context, "SET %s %b", key_string.c_str(), value.data(), value.size());
+ LOG("Redis SET {} [{} bytes]", key_string, value.size());
+ const auto reply =
+ redis_command("SET %s %b", key_string.c_str(), value.data(), value.size());
if (!reply) {
- LOG("Failed to put {} to Redis (NULL)", key_string);
- } else if (reply->type == REDIS_REPLY_STATUS) {
+ return nonstd::make_unexpected(reply.error());
+ } else if ((*reply)->type == REDIS_REPLY_STATUS) {
return true;
- } else if (reply->type == REDIS_REPLY_ERROR) {
- LOG("Failed to put {} to Redis: {}", key_string, reply->str);
} else {
- LOG("Failed to put {} to Redis: unknown reply type {}",
- key_string,
- reply->type);
+ LOG("Unknown reply type: {}", (*reply)->type);
+ return nonstd::make_unexpected(Failure::error);
}
+}
- return nonstd::make_unexpected(Error::error);
+nonstd::expected<bool, SecondaryStorage::Backend::Failure>
+RedisStorageBackend::remove(const Digest& key)
+{
+ const auto key_string = get_key_string(key);
+ LOG("Redis DEL {}", key_string);
+ const auto reply = redis_command("DEL %s", key_string.c_str());
+ if (!reply) {
+ return nonstd::make_unexpected(reply.error());
+ } else if ((*reply)->type == REDIS_REPLY_INTEGER) {
+ return (*reply)->integer > 0;
+ } else {
+ LOG("Unknown reply type: {}", (*reply)->type);
+ return nonstd::make_unexpected(Failure::error);
+ }
}
-nonstd::expected<bool, SecondaryStorage::Error>
-RedisStorage::remove(const Digest& key)
+void
+RedisStorageBackend::connect(const Url& url,
+ const uint32_t connect_timeout,
+ const uint32_t operation_timeout)
{
- const int err = connect();
- if (is_timeout(err)) {
- return nonstd::make_unexpected(Error::timeout);
- } else if (is_error(err)) {
- return nonstd::make_unexpected(Error::error);
+ const std::string host = url.host().empty() ? "localhost" : url.host();
+ const uint32_t port = url.port().empty()
+ ? DEFAULT_PORT
+ : util::value_or_throw<Fatal>(
+ util::parse_unsigned(url.port(), 1, 65535, "port"));
+ ASSERT(url.path().empty() || url.path()[0] == '/');
+
+ LOG("Redis connecting to {}:{} (connect timeout {} ms)",
+ url.host(),
+ port,
+ connect_timeout);
+ m_context.reset(redisConnectWithTimeout(
+ url.host().c_str(), port, to_timeval(connect_timeout)));
+
+ if (!m_context) {
+ throw Failed("Redis context construction error");
+ }
+ if (is_timeout(m_context->err)) {
+ throw Failed(FMT("Redis connection timeout: {}", m_context->errstr),
+ Failure::timeout);
+ }
+ if (is_error(m_context->err)) {
+ throw Failed(FMT("Redis connection error: {}", m_context->errstr));
}
- const std::string key_string = get_key_string(key);
- LOG("Redis DEL {}", key_string);
+ LOG("Redis operation timeout set to {} ms", operation_timeout);
+ if (redisSetTimeout(m_context.get(), to_timeval(operation_timeout))
+ != REDIS_OK) {
+ throw Failed("Failed to set operation timeout");
+ }
+
+ LOG_RAW("Redis connection OK");
+}
+
+void
+RedisStorageBackend::select_database(const Url& url)
+{
+ const uint32_t db_number =
+ url.path().empty() ? 0
+ : util::value_or_throw<Fatal>(util::parse_unsigned(
+ url.path().substr(1),
+ 0,
+ std::numeric_limits<uint32_t>::max(),
+ "db number"));
- const auto reply = redis_command(m_context, "DEL %s", key_string.c_str());
+ if (db_number != 0) {
+ LOG("Redis SELECT {}", db_number);
+ const auto reply =
+ util::value_or_throw<Failed>(redis_command("SELECT %d", db_number));
+ }
+}
+
+void
+RedisStorageBackend::authenticate(const Url& url)
+{
+ const auto password_username_pair = split_user_info(url.user_info());
+ const auto& password = password_username_pair.first;
+ if (password) {
+ decltype(redis_command("")) reply = nonstd::make_unexpected(Failure::error);
+ const auto& username = password_username_pair.second;
+ if (username) {
+ LOG("Redis AUTH {} {}", *username, k_redacted_password);
+ reply = util::value_or_throw<Failed>(
+ redis_command("AUTH %s %s", username->c_str(), password->c_str()));
+ } else {
+ LOG("Redis AUTH {}", k_redacted_password);
+ reply = util::value_or_throw<Failed>(
+ redis_command("AUTH %s", password->c_str()));
+ }
+ }
+}
+
+nonstd::expected<RedisReply, SecondaryStorage::Backend::Failure>
+RedisStorageBackend::redis_command(const char* format, ...)
+{
+ va_list ap;
+ va_start(ap, format);
+ auto reply =
+ static_cast<redisReply*>(redisvCommand(m_context.get(), format, ap));
+ va_end(ap);
if (!reply) {
- LOG("Failed to remove {} from Redis (NULL)", key_string);
- } else if (reply->type == REDIS_REPLY_INTEGER) {
- return reply->integer > 0;
+ LOG("Redis command failed: {}", m_context->errstr);
+ return nonstd::make_unexpected(is_timeout(m_context->err) ? Failure::timeout
+ : Failure::error);
} else if (reply->type == REDIS_REPLY_ERROR) {
- LOG("Failed to remove {} from Redis: {}", key_string, reply->str);
+ LOG("Redis command failed: {}", reply->str);
+ return nonstd::make_unexpected(Failure::error);
} else {
- LOG("Failed to remove {} from Redis: unknown reply type {}",
- key_string,
- reply->type);
+ return RedisReply(reply, freeReplyObject);
}
-
- return nonstd::make_unexpected(Error::error);
}
std::string
-RedisStorage::get_key_string(const Digest& digest) const
+RedisStorageBackend::get_key_string(const Digest& digest) const
{
return FMT("{}:{}", m_prefix, digest.to_string());
}
+} // namespace
+
+std::unique_ptr<SecondaryStorage::Backend>
+RedisStorage::create_backend(const Backend::Params& params) const
+{
+ return std::make_unique<RedisStorageBackend>(params);
+}
+
+void
+RedisStorage::redact_secrets(Backend::Params& params) const
+{
+ auto& url = params.url;
+ const auto user_info = util::split_once(url.user_info(), ':');
+ if (user_info.second) {
+ // redis://username:password@host
+ url.user_info(FMT("{}:{}", user_info.first, k_redacted_password));
+ } else if (!user_info.first.empty()) {
+ // redis://password@host
+ url.user_info(k_redacted_password);
+ }
+}
+
} // namespace secondary
} // namespace storage
#pragma once
-#include "storage/SecondaryStorage.hpp"
-#include "storage/types.hpp"
-
-#include <third_party/url.hpp>
-
-struct redisContext;
+#include "SecondaryStorage.hpp"
namespace storage {
namespace secondary {
-class RedisStorage : public storage::SecondaryStorage
+class RedisStorage : public SecondaryStorage
{
public:
- RedisStorage(const Url& url, const AttributeMap& attributes);
- ~RedisStorage();
-
- nonstd::expected<nonstd::optional<std::string>, Error>
- get(const Digest& key) override;
- nonstd::expected<bool, Error> put(const Digest& key,
- const std::string& value,
- bool only_if_missing) override;
- nonstd::expected<bool, Error> remove(const Digest& key) override;
-
-private:
- Url m_url;
- std::string m_prefix;
- redisContext* m_context;
- const uint64_t m_connect_timeout;
- const uint64_t m_operation_timeout;
- bool m_connected;
- bool m_invalid;
+ std::unique_ptr<Backend>
+ create_backend(const Backend::Params& params) const override;
- int connect();
- int auth();
- std::string get_key_string(const Digest& digest) const;
+ void redact_secrets(Backend::Params& params) const override;
};
} // namespace secondary
--- /dev/null
+// Copyright (C) 2021 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "SecondaryStorage.hpp"
+
+#include <util/expected.hpp>
+#include <util/string.hpp>
+
+namespace storage {
+namespace secondary {
+
+bool
+SecondaryStorage::Backend::is_framework_attribute(const std::string& name)
+{
+ return name == "read-only";
+}
+
+std::chrono::milliseconds
+SecondaryStorage::Backend::parse_timeout_attribute(const std::string& value)
+{
+ return std::chrono::milliseconds(util::value_or_throw<Failed>(
+ util::parse_unsigned(value, 1, 60 * 1000, "timeout")));
+}
+
+} // namespace secondary
+} // namespace storage
--- /dev/null
+// Copyright (C) 2021 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#pragma once
+
+#include <storage/types.hpp>
+
+#include <third_party/nonstd/expected.hpp>
+#include <third_party/nonstd/optional.hpp>
+#include <third_party/url.hpp>
+
+#include <chrono>
+#include <memory>
+#include <string>
+#include <vector>
+
+class Digest;
+
+namespace storage {
+namespace secondary {
+
+constexpr auto k_redacted_password = "********";
+const auto k_default_connect_timeout = std::chrono::milliseconds{100};
+const auto k_default_operation_timeout = std::chrono::milliseconds{10000};
+
+// This class defines the API that a secondary storage must implement.
+class SecondaryStorage
+{
+public:
+ class Backend
+ {
+ public:
+ struct Attribute
+ {
+ std::string key; // Key part.
+ std::string value; // Value part, percent-decoded.
+ std::string raw_value; // Value part, not percent-decoded.
+ };
+
+ struct Params
+ {
+ Url url;
+ std::vector<Attribute> attributes;
+ };
+
+ enum class Failure {
+ error, // Operation error, e.g. bad parameters or failed connection.
+ timeout, // Timeout, e.g. due to slow network or server.
+ };
+
+ class Failed : public std::runtime_error
+ {
+ public:
+ Failed(Failure failure);
+ Failed(const std::string& message, Failure failure = Failure::error);
+
+ Failure failure() const;
+
+ private:
+ Failure m_failure;
+ };
+
+ virtual ~Backend() = default;
+
+ // Get the value associated with `key`. Returns the value on success or
+ // nonstd::nullopt if the entry is not present.
+ virtual nonstd::expected<nonstd::optional<std::string>, Failure>
+ get(const Digest& key) = 0;
+
+ // Put `value` associated to `key` in the storage. A true `only_if_missing`
+ // is a hint that the value does not have to be set if already present.
+ // Returns true if the entry was stored, otherwise false.
+ virtual nonstd::expected<bool, Failure>
+ put(const Digest& key,
+ const std::string& value,
+ bool only_if_missing = false) = 0;
+
+ // Remove `key` and its associated value. Returns true if the entry was
+ // removed, otherwise false.
+ virtual nonstd::expected<bool, Failure> remove(const Digest& key) = 0;
+
+ // Determine whether an attribute is handled by the secondary storage
+ // framework itself.
+ static bool is_framework_attribute(const std::string& name);
+
+ // Parse a timeout `value`, throwing `Failed` on error.
+ static std::chrono::milliseconds
+ parse_timeout_attribute(const std::string& value);
+ };
+
+ virtual ~SecondaryStorage() = default;
+
+ // Create an instance of the backend. The instance is created just before the
+ // first call to a backend method, so the backend constructor can open a
+ // connection or similar right away if wanted. The method should throw `Fatal`
+ // on fatal configuration error or `Backend::Failed` on connection error or
+ // timeout.
+ virtual std::unique_ptr<Backend>
+ create_backend(const Backend::Params& parameters) const = 0;
+
+ // Redact secrets in backend parameters, if any.
+ virtual void redact_secrets(Backend::Params& parameters) const;
+};
+
+// --- Inline implementations ---
+
+inline void
+SecondaryStorage::redact_secrets(
+ SecondaryStorage::Backend::Params& /*config*/) const
+{
+}
+
+inline SecondaryStorage::Backend::Failed::Failed(Failure failure)
+ : Failed("", failure)
+{
+}
+
+inline SecondaryStorage::Backend::Failed::Failed(const std::string& message,
+ Failure failure)
+ : std::runtime_error::runtime_error(message),
+ m_failure(failure)
+{
+}
+
+inline SecondaryStorage::Backend::Failure
+SecondaryStorage::Backend::Failed::failure() const
+{
+ return m_failure;
+}
+
+} // namespace secondary
+} // namespace storage
#include <functional>
#include <string>
-#include <unordered_map>
namespace storage {
-using AttributeMap =
- std::unordered_map<std::string /*key*/, std::string /*value*/>;
using CacheEntryWriter = std::function<bool(const std::string& path)>;
} // namespace storage
# -------------------------------------------------------------------------
TEST "Basic auth"
- start_http_server 12780 secondary "somebody:secret"
- export CCACHE_SECONDARY_STORAGE="http://somebody:secret@localhost:12780"
+ start_http_server 12780 secondary "somebody:secret123"
+ export CCACHE_SECONDARY_STORAGE="http://somebody:secret123@localhost:12780"
- $CCACHE_COMPILE -c test.c
+ CCACHE_DEBUG=1 $CCACHE_COMPILE -c test.c
expect_stat 'cache hit (direct)' 0
expect_stat 'cache miss' 1
expect_stat 'files in cache' 2
expect_file_count 2 '*' secondary # result + manifest
+ expect_not_contains test.o.ccache-log secret123
# -------------------------------------------------------------------------
TEST "Basic auth required"
- start_http_server 12780 secondary "somebody:secret"
+ start_http_server 12780 secondary "somebody:secret123"
# no authentication configured on client
export CCACHE_SECONDARY_STORAGE="http://localhost:12780"
# -------------------------------------------------------------------------
TEST "Basic auth failed"
- start_http_server 12780 secondary "somebody:secret"
+ start_http_server 12780 secondary "somebody:secret123"
export CCACHE_SECONDARY_STORAGE="http://somebody:wrong@localhost:12780"
CCACHE_DEBUG=1 $CCACHE_COMPILE -c test.c
expect_stat 'cache miss' 1
expect_stat 'files in cache' 2
expect_file_count 0 '*' secondary # result + manifest
+ expect_not_contains test.o.ccache-log secret123
expect_contains test.o.ccache-log "status code: 401"
# -------------------------------------------------------------------------
TEST "Password"
port=7777
- password=secret
+ password=secret123
redis_url="redis://${password}@localhost:${port}"
export CCACHE_SECONDARY_STORAGE="${redis_url}"
start_redis_server "${port}" "${password}"
- $CCACHE_COMPILE -c test.c
+ CCACHE_DEBUG=1 $CCACHE_COMPILE -c test.c
expect_stat 'cache hit (direct)' 0
expect_stat 'cache miss' 1
expect_stat 'files in cache' 2
expect_number_of_redis_cache_entries 2 "$redis_url" # result + manifest
+ expect_not_contains test.o.ccache-log "${password}"
$CCACHE_COMPILE -c test.c
expect_stat 'cache hit (direct)' 1
projects / thirdparty / ccache.git / commitdiff
