Fixes #1321.
return util::join(features, " ");
}
+// Representation of one shard configuration.
struct RemoteStorageShardConfig
{
std::string name;
double weight;
+ Url url; // Cache of URL with expanded "*"
};
+// Representation of one entry in the remote_storage config option.
struct RemoteStorageConfig
{
+ // Raw URL with unexpanded "*".
+ std::string url_str;
+
+ // "shard" attribute.
std::vector<RemoteStorageShardConfig> shards;
- remote::RemoteStorage::Backend::Params params;
+
+ // "read-only" attribute.
bool read_only = false;
+
+ // Other attributes.
+ std::vector<remote::RemoteStorage::Backend::Attribute> attributes;
};
+// An instantiated remote storage backend.
struct RemoteStorageBackendEntry
{
- Url url; // With expanded "*".
- std::string url_for_logging; // With expanded "*".
+ Url url; // With expanded "*"
+ std::string url_for_logging; // With expanded "*"
std::unique_ptr<remote::RemoteStorage::Backend> impl;
bool failed = false;
};
+// An instantiated remote storage.
struct RemoteStorageEntry
{
RemoteStorageConfig config;
- std::string url_for_logging; // With unexpanded "*".
std::shared_ptr<remote::RemoteStorage> storage;
std::vector<RemoteStorageBackendEntry> backends;
};
static std::string
to_string(const RemoteStorageConfig& entry)
{
- std::string result = entry.params.url.str();
- for (const auto& attr : entry.params.attributes) {
+ std::string result = entry.url_str;
+ for (const auto& attr : entry.attributes) {
result += FMT("|{}={}", attr.key, attr.raw_value);
}
return result;
}
-static Url
+static tl::expected<Url, std::string>
url_from_string(const std::string& url_string)
{
// The Url class is parsing the URL object lazily. Check if the URL is valid
// now to avoid exceptions later.
+ Url url(url_string);
try {
- Url url(url_string);
- std::ignore = url.str();
- return url;
+ std::ignore = url.scheme();
} catch (const std::exception& e) {
- throw core::Error(FMT("Cannot parse URL {}: {}", url_string, e.what()));
+ return tl::unexpected(FMT("Cannot parse URL {}: {}", url_string, e.what()));
+ }
+ if (url.scheme().empty()) {
+ return tl::unexpected(FMT("URL scheme must not be empty: {}", url_string));
}
+ return url;
}
static RemoteStorageConfig
}
RemoteStorageConfig result;
- const auto url_str = std::string(parts[0]);
- result.params.url = url_from_string(url_str);
-
- if (result.params.url.scheme().empty()) {
- throw core::Error(FMT("URL scheme must not be empty: {}", entry));
- }
+ result.url_str = std::string(parts[0]);
+ const auto& url_str = result.url_str;
for (size_t i = 1; i < parts.size(); ++i) {
if (parts[i].empty()) {
if (key == "read-only") {
result.read_only = (value == "true");
} else if (key == "shards") {
- if (url_str.find('*') == std::string::npos) {
+ const auto asterisk_count =
+ std::count(url_str.begin(), url_str.end(), '*');
+ if (asterisk_count == 0) {
throw core::Error(
FMT(R"(Missing "*" in URL when using shards: "{}")", url_str));
+ } else if (asterisk_count > 1) {
+ throw core::Error(
+ FMT(R"(Multiple "*" in URL when using shards: "{}")", url_str));
}
+ std::string scheme;
for (const auto& shard : util::Tokenizer(value, ",")) {
double weight = 1.0;
std::string_view name;
name = shard;
}
- result.shards.push_back({std::string(name), weight});
+ Url url = util::value_or_throw<core::Error>(
+ url_from_string(util::replace_first(url_str, "*", name)));
+ if (!scheme.empty() && url.scheme() != scheme) {
+ throw core::Error(FMT("Scheme {} different from {} in {}",
+ url.scheme(),
+ scheme,
+ url_str));
+ }
+ result.shards.push_back({std::string(name), weight, url});
}
}
- result.params.attributes.push_back(
+ result.attributes.push_back(
{std::string(key), value, std::string(raw_value)});
}
+ // No shards => save the single URL as the sole shard.
+ if (result.shards.empty()) {
+ result.shards.push_back(
+ {"", 0.0, util::value_or_throw<core::Error>(url_from_string(url_str))});
+ }
+
return result;
}
}
static std::shared_ptr<remote::RemoteStorage>
-get_storage(const Url& url)
+get_storage(const std::string& scheme)
{
- const auto it = k_remote_storage_implementations.find(url.scheme());
+ const auto it = k_remote_storage_implementations.find(scheme);
if (it != k_remote_storage_implementations.end()) {
return it->second;
} else {
return !m_remote_storages.empty();
}
+static std::string
+get_redacted_url_str_for_logging(const Url& url)
+{
+ Url redacted_url(url);
+ if (!url.user_info().empty()) {
+ redacted_url.user_info(k_redacted_password);
+ }
+ return redacted_url.str();
+}
+
std::string
Storage::get_remote_storage_config_for_logging() const
{
auto configs = parse_storage_configs(m_config.remote_storage());
for (auto& config : configs) {
- const auto storage = get_storage(config.params.url);
- if (storage) {
- storage->redact_secrets(config.params);
- }
+ const auto url = url_from_string(config.url_str);
+ if (url) {
+ const auto storage = get_storage(url->scheme());
+ if (storage) {
+ config.url_str = get_redacted_url_str_for_logging(*url);
+ storage->redact_secrets(config.attributes);
+ }
+ } // else: unexpanded URL is not a proper URL, not much we can do
}
return util::join(configs, " ");
}
-static void
-redact_url_for_logging(Url& url_for_logging)
-{
- url_for_logging.user_info("");
-}
-
void
Storage::add_remote_storages()
{
const auto configs = parse_storage_configs(m_config.remote_storage());
for (const auto& config : configs) {
- auto url_for_logging = config.params.url;
- redact_url_for_logging(url_for_logging);
- const auto storage = get_storage(config.params.url);
+ ASSERT(!config.shards.empty());
+ const std::string scheme = config.shards.front().url.scheme();
+ const auto storage = get_storage(scheme);
if (!storage) {
- throw core::Error(
- FMT("unknown remote storage URL: {}", url_for_logging.str()));
+ throw core::Error(FMT("unknown remote storage scheme: {}", scheme));
}
m_remote_storages.push_back(std::make_unique<RemoteStorageEntry>(
- RemoteStorageEntry{config, url_for_logging.str(), storage, {}}));
+ RemoteStorageEntry{config, storage, {}}));
}
}
static Url
get_shard_url(const Hash::Digest& key,
- const std::string& url,
const std::vector<RemoteStorageShardConfig>& shards)
{
ASSERT(!shards.empty());
+ if (shards.size() == 1) {
+ return shards.front().url;
+ }
+
// This is the "weighted rendezvous hashing" algorithm.
double highest_score = -1.0;
- std::string best_shard;
+ Url best_shard_url;
for (const auto& shard_config : shards) {
util::XXH3_64 hash;
hash.update(key.data(), key.size());
const double weighted_score =
score == 0.0 ? 0.0 : shard_config.weight / -std::log(score);
if (weighted_score > highest_score) {
- best_shard = shard_config.name;
+ best_shard_url = shard_config.url;
highest_score = weighted_score;
}
}
- return url_from_string(util::replace_first(url, "*", best_shard));
+ return best_shard_url;
}
RemoteStorageBackendEntry*
const bool for_writing)
{
if (for_writing && entry.config.read_only) {
- LOG("Not {} {} since it is read-only",
+ LOG("Not {} {} storage since it is read-only",
operation_description,
- entry.url_for_logging);
+ entry.config.shards.front().url.scheme());
return nullptr;
}
- const auto shard_url =
- entry.config.shards.empty()
- ? entry.config.params.url
- : get_shard_url(key, entry.config.params.url.str(), entry.config.shards);
+ const auto shard_url = get_shard_url(key, entry.config.shards);
+ const auto url_str_for_logging =
+ get_redacted_url_str_for_logging(shard_url.str());
auto backend =
std::find_if(entry.backends.begin(),
entry.backends.end(),
[&](const auto& x) { return x.url.str() == shard_url.str(); });
if (backend == entry.backends.end()) {
- auto shard_url_for_logging = shard_url;
- redact_url_for_logging(shard_url_for_logging);
- entry.backends.push_back(
- {shard_url, shard_url_for_logging.str(), {}, false});
- auto shard_params = entry.config.params;
- shard_params.url = shard_url;
+ entry.backends.push_back({shard_url, url_str_for_logging, {}, false});
try {
- entry.backends.back().impl = entry.storage->create_backend(shard_params);
+ entry.backends.back().impl =
+ entry.storage->create_backend(shard_url, entry.config.attributes);
} catch (const remote::RemoteStorage::Backend::Failed& e) {
LOG("Failed to construct backend for {}{}",
- entry.url_for_logging,
+ url_str_for_logging,
std::string_view(e.what()).empty() ? "" : FMT(": {}", e.what()));
mark_backend_as_failed(entry.backends.back(), e.failure());
return nullptr;
} else if (backend->failed) {
LOG("Not {} {} since it failed earlier",
operation_description,
- entry.url_for_logging);
+ url_str_for_logging);
return nullptr;
} else {
return &*backend;
namespace storage {
+constexpr auto k_redacted_password = "********";
+
std::string get_features();
struct RemoteStorageBackendEntry;
class FileStorageBackend : public RemoteStorage::Backend
{
public:
- FileStorageBackend(const Params& params);
+ FileStorageBackend(const Url& url,
+ const std::vector<Backend::Attribute>& attributes);
tl::expected<std::optional<util::Bytes>, Failure>
get(const Hash::Digest& key) override;
std::string get_entry_path(const Hash::Digest& key) const;
};
-FileStorageBackend::FileStorageBackend(const Params& params)
+FileStorageBackend::FileStorageBackend(
+ const Url& url, const std::vector<Backend::Attribute>& attributes)
{
- ASSERT(params.url.scheme() == "file");
+ ASSERT(url.scheme() == "file");
- const auto& host = params.url.host();
+ const auto& host = url.host();
#ifdef _WIN32
- m_dir = util::replace_all(params.url.path(), "/", "\\");
+ m_dir = util::replace_all(url.path(), "/", "\\");
if (m_dir.length() >= 3 && m_dir[0] == '\\' && m_dir[2] == ':') {
// \X:\foo\bar -> X:\foo\bar according to RFC 8089 appendix E.2.
m_dir = m_dir.substr(1);
throw core::Fatal(
FMT("invalid file URL \"{}\": specifying a host other than localhost is"
" not supported",
- params.url.str()));
+ url.str()));
}
- m_dir = params.url.path();
+ m_dir = url.path();
#endif
- for (const auto& attr : params.attributes) {
+ for (const auto& attr : attributes) {
if (attr.key == "layout") {
if (attr.value == "flat") {
m_layout = Layout::flat;
} // namespace
std::unique_ptr<RemoteStorage::Backend>
-FileStorage::create_backend(const Backend::Params& params) const
+FileStorage::create_backend(
+ const Url& url, const std::vector<Backend::Attribute>& attributes) const
{
- return std::make_unique<FileStorageBackend>(params);
+ return std::make_unique<FileStorageBackend>(url, attributes);
}
} // namespace storage::remote
-// Copyright (C) 2021-2022 Joel Rosdahl and other contributors
+// Copyright (C) 2021-2023 Joel Rosdahl and other contributors
//
// See doc/AUTHORS.adoc for a complete list of contributors.
//
class FileStorage : public RemoteStorage
{
public:
- std::unique_ptr<Backend>
- create_backend(const Backend::Params& params) const override;
+ std::unique_ptr<Backend> create_backend(
+ const Url& url,
+ const std::vector<Backend::Attribute>& attributes) const override;
};
} // namespace storage::remote
#include <Hash.hpp>
#include <ccache.hpp>
#include <core/exceptions.hpp>
+#include <storage/Storage.hpp>
#include <util/assertions.hpp>
#include <util/expected.hpp>
#include <util/fmtmacros.hpp>
class HttpStorageBackend : public RemoteStorage::Backend
{
public:
- HttpStorageBackend(const Params& params);
+ HttpStorageBackend(const Url& url,
+ const std::vector<Backend::Attribute>& attributes);
tl::expected<std::optional<util::Bytes>, Failure>
get(const Hash::Digest& key) override;
: RemoteStorage::Backend::Failure::error;
}
-HttpStorageBackend::HttpStorageBackend(const Params& params)
- : m_url_path(get_url_path(params.url)),
- m_http_client(get_url(params.url))
+HttpStorageBackend::HttpStorageBackend(
+ const Url& url, const std::vector<Backend::Attribute>& attributes)
+ : m_url_path(get_url_path(url)),
+ m_http_client(get_url(url))
{
- if (!params.url.user_info().empty()) {
- const auto [user, password] = util::split_once(params.url.user_info(), ':');
+ if (!url.user_info().empty()) {
+ const auto [user, password] = util::split_once(url.user_info(), ':');
if (!password) {
throw core::Fatal(FMT("Expected username:password in URL but got \"{}\"",
- params.url.user_info()));
+ url.user_info()));
}
m_http_client.set_basic_auth(std::string(user), std::string(*password));
}
auto connect_timeout = k_default_connect_timeout;
auto operation_timeout = k_default_operation_timeout;
- for (const auto& attr : params.attributes) {
+ for (const auto& attr : attributes) {
if (attr.key == "bearer-token") {
m_http_client.set_bearer_token_auth(attr.value);
} else if (attr.key == "connect-timeout") {
} // namespace
std::unique_ptr<RemoteStorage::Backend>
-HttpStorage::create_backend(const Backend::Params& params) const
+HttpStorage::create_backend(
+ const Url& url, const std::vector<Backend::Attribute>& attributes) const
{
- return std::make_unique<HttpStorageBackend>(params);
+ return std::make_unique<HttpStorageBackend>(url, attributes);
}
void
-HttpStorage::redact_secrets(Backend::Params& params) const
+HttpStorage::redact_secrets(std::vector<Backend::Attribute>& attributes) const
{
- auto& url = params.url;
- const auto [user, password] = util::split_once(url.user_info(), ':');
- if (password) {
- url.user_info(FMT("{}:{}", user, k_redacted_password));
- }
-
auto bearer_token_attribute =
- std::find_if(params.attributes.begin(),
- params.attributes.end(),
- [&](const auto& attr) { return attr.key == "bearer-token"; });
- if (bearer_token_attribute != params.attributes.end()) {
- bearer_token_attribute->value = k_redacted_password;
- bearer_token_attribute->raw_value = k_redacted_password;
+ std::find_if(attributes.begin(), attributes.end(), [&](const auto& attr) {
+ return attr.key == "bearer-token";
+ });
+ if (bearer_token_attribute != attributes.end()) {
+ bearer_token_attribute->value = storage::k_redacted_password;
+ bearer_token_attribute->raw_value = storage::k_redacted_password;
}
}
class HttpStorage : public RemoteStorage
{
public:
- std::unique_ptr<Backend>
- create_backend(const Backend::Params& params) const override;
+ std::unique_ptr<Backend> create_backend(
+ const Url& url,
+ const std::vector<Backend::Attribute>& attributes) const override;
- void redact_secrets(Backend::Params& params) const override;
+ void
+ redact_secrets(std::vector<Backend::Attribute>& attributes) const override;
};
} // namespace storage::remote
#include <Hash.hpp>
#include <core/exceptions.hpp>
+#include <storage/Storage.hpp>
#include <util/assertions.hpp>
#include <util/expected.hpp>
#include <util/fmtmacros.hpp>
class RedisStorageBackend : public RemoteStorage::Backend
{
public:
- RedisStorageBackend(const RemoteStorage::Backend::Params& params);
+ RedisStorageBackend(const Url& url,
+ const std::vector<Backend::Attribute>& attributes);
tl::expected<std::optional<util::Bytes>, Failure>
get(const Hash::Digest& key) override;
}
}
-RedisStorageBackend::RedisStorageBackend(const Params& params)
+RedisStorageBackend::RedisStorageBackend(
+ const Url& url,
+ const std::vector<Backend::Attribute>& attributes)
: m_prefix("ccache"), // TODO: attribute
m_context(nullptr, redisFree)
{
- const auto& url = params.url;
ASSERT(url.scheme() == "redis" || url.scheme() == "redis+unix");
- if (url.scheme() == "redis+unix" && !params.url.host().empty()
- && params.url.host() != "localhost") {
+ if (url.scheme() == "redis+unix" && !url.host().empty()
+ && url.host() != "localhost") {
throw core::Fatal(
FMT("invalid file path \"{}\": specifying a host other than localhost is"
" not supported",
- params.url.str(),
- params.url.host()));
+ url.str(),
+ url.host()));
}
auto connect_timeout = k_default_connect_timeout;
auto operation_timeout = k_default_operation_timeout;
- for (const auto& attr : params.attributes) {
+ for (const auto& attr : attributes) {
if (attr.key == "connect-timeout") {
connect_timeout = parse_timeout_attribute(attr.value);
} else if (attr.key == "operation-timeout") {
if (password) {
if (user) {
// redis://user:password@host
- LOG("Redis AUTH {} {}", *user, k_redacted_password);
+ LOG("Redis AUTH {} {}", *user, storage::k_redacted_password);
util::value_or_throw<Failed>(
redis_command("AUTH %s %s", user->c_str(), password->c_str()));
} else {
// redis://password@host
- LOG("Redis AUTH {}", k_redacted_password);
+ LOG("Redis AUTH {}", storage::k_redacted_password);
util::value_or_throw<Failed>(redis_command("AUTH %s", password->c_str()));
}
}
} // namespace
std::unique_ptr<RemoteStorage::Backend>
-RedisStorage::create_backend(const Backend::Params& params) const
+RedisStorage::create_backend(
+ const Url& url, const std::vector<Backend::Attribute>& attributes) const
{
- return std::make_unique<RedisStorageBackend>(params);
-}
-
-void
-RedisStorage::redact_secrets(Backend::Params& params) const
-{
- auto& url = params.url;
- const auto [user, password] = split_user_info(url.user_info());
- if (password) {
- if (user) {
- // redis://user:password@host
- url.user_info(FMT("{}:{}", *user, k_redacted_password));
- } else {
- // redis://password@host
- url.user_info(k_redacted_password);
- }
- }
+ return std::make_unique<RedisStorageBackend>(url, attributes);
}
} // namespace storage::remote
-// Copyright (C) 2021-2022 Joel Rosdahl and other contributors
+// Copyright (C) 2021-2023 Joel Rosdahl and other contributors
//
// See doc/AUTHORS.adoc for a complete list of contributors.
//
class RedisStorage : public RemoteStorage
{
public:
- std::unique_ptr<Backend>
- create_backend(const Backend::Params& params) const override;
-
- void redact_secrets(Backend::Params& params) const override;
+ std::unique_ptr<Backend> create_backend(
+ const Url& url,
+ const std::vector<Backend::Attribute>& attributes) const override;
};
} // namespace storage::remote
namespace storage::remote {
-constexpr auto k_redacted_password = "********";
const auto k_default_connect_timeout = std::chrono::milliseconds{100};
const auto k_default_operation_timeout = std::chrono::milliseconds{10000};
std::string raw_value; // Value part, not percent-decoded.
};
- struct Params
- {
- Url url;
- std::vector<Attribute> attributes;
- };
-
enum class Failure {
error, // Operation error, e.g. bad parameters or failed connection.
timeout, // Timeout, e.g. due to slow network or server.
// `core::Fatal` on fatal configuration error or `Backend::Failed` on
// connection error or timeout.
virtual std::unique_ptr<Backend>
- create_backend(const Backend::Params& parameters) const = 0;
+ create_backend(const Url& url,
+ const std::vector<Backend::Attribute>& attributes) const = 0;
- // Redact secrets in backend parameters, if any.
- virtual void redact_secrets(Backend::Params& parameters) const;
+ // Redact secrets in backend attributes, if any.
+ virtual void
+ redact_secrets(std::vector<Backend::Attribute>& attributes) const;
};
// --- Inline implementations ---
inline void
-RemoteStorage::redact_secrets(RemoteStorage::Backend::Params& /*config*/) const
+RemoteStorage::redact_secrets(
+ std::vector<Backend::Attribute>& /*attributes*/) const
{
}
test_failed "Expected remote/a or remote/b to exist"
fi
+ $CCACHE -Cz >/dev/null
+ rm -rf remote
+
+ CCACHE_REMOTE_STORAGE="*|shards=file://$PWD/remote/a,file://$PWD/remote/b"
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 0
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2
+ if [ ! -d remote/a ] && [ ! -d remote/b ]; then
+ test_failed "Expected remote/a or remote/b to exist"
+ fi
+
# -------------------------------------------------------------------------
TEST "Reshare"
expect_not_contains test.o.*.ccache-log secret123
expect_contains test.o.*.ccache-log "status code: 401"
fi
+
# -------------------------------------------------------------------------
+ TEST "Port sharding"
+
+ start_http_server 12780 remote
+ export CCACHE_REMOTE_STORAGE="http://localhost:*|shards=12780"
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 0
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2
+ expect_file_count 2 '*' remote # result + manifest
+ subdirs=$(find remote -type d | wc -l)
+ if [ "${subdirs}" -lt 2 ]; then # "remote" itself counts as one
+ test_failed "Expected subdirectories in remote"
+ fi
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 1
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2
+ expect_file_count 2 '*' remote # result + manifest
+
+ # -------------------------------------------------------------------------
TEST "IPv6 address"
if maybe_start_ipv6_http_server 12780 remote; then
projects / thirdparty / ccache.git / commitdiff
