feat: Add "exists" operation to remote storage helper protocol

In reshare mode, ccache now asks the remote storage helper if a value
exists before putting the value. This avoids sending the full payload to
the helper in case the value already exists.

diff --git a/doc/remote_storage_helper_spec.md b/doc/remote_storage_helper_spec.md
index 94f5dce4770f5cdffa17279d6dbfd6c2d8918e18..34b3c812c5f5543d6549fe829533589b4a35fca6 100644 (file)
--- a/doc/remote_storage_helper_spec.md
+++ b/doc/remote_storage_helper_spec.md
@@ -83,6 +83,7 @@ This is a specification of the custom binary IPC protocol between ccache
 - `<byte>`: 1 byte
 - `<u8>`: unsigned 8-bit integer (1 byte)
 - `<u64>`: unsigned 64-bit integer in host byte order (8 bytes)
+- `<bool>`: 1 byte (0x00: false, 0x01: true)
 
 ### Common types
 
@@ -108,6 +109,7 @@ This is a specification of the custom binary IPC protocol between ccache
 
 - 0x00: `get`/`put`/`remove` requests
 - 0x01: `info` request
+- 0x02: `exists` request
 
 ### Server greeting (server to client)
 
@@ -181,3 +183,13 @@ Get information about the helper.
 <diag_num>        ::= <u8>
 <diag>            ::= <msg>              ; message to be logged by the client
 ```
+
+#### Exists (capability 0x02)
+
+Check if remote storage has a value.
+
+```
+<exists_request>  ::= 0x05 <key>
+<exists_response> ::= <ok> <exists> | <err>
+<exists>          ::= <bool>
+```

diff --git a/src/ccache/storage/remote/client.cpp b/src/ccache/storage/remote/client.cpp
index 6631782749872cfd7ab1ec35c37899497d183f2f..633e310fa11737f633f7813700580e748fc429ff 100644 (file)
--- a/src/ccache/storage/remote/client.cpp
+++ b/src/ccache/storage/remote/client.cpp
@@ -33,6 +33,7 @@ constexpr uint8_t k_request_put = 0x01;
 constexpr uint8_t k_request_remove = 0x02;
 constexpr uint8_t k_request_stop = 0x03;
 constexpr uint8_t k_request_info = 0x04;
+constexpr uint8_t k_request_exists = 0x05;
 
 } // namespace
 
@@ -133,6 +134,23 @@ Client::verify_key_size(std::span<const uint8_t> key)
   return {};
 }
 
+tl::expected<bool, Client::Error>
+Client::exists(std::span<const uint8_t> key)
+{
+  TRY(verify_connected());
+  TRY(verify_key_size(key));
+
+  m_request_start_time = std::chrono::steady_clock::now();
+
+  util::Bytes msg;
+  msg.reserve(2 + key.size());
+  msg.push_back(k_request_exists);
+  msg.push_back(static_cast<uint8_t>(key.size()));
+  msg.insert(msg.end(), key.data(), key.size());
+  TRY(send_bytes(msg));
+  return receive_response_exists();
+}
+
 tl::expected<std::optional<util::Bytes>, Client::Error>
 Client::get(std::span<const uint8_t> key)
 {
@@ -203,7 +221,7 @@ Client::put(std::span<const uint8_t> key,
   header.insert(header.end(), len_bytes, sizeof(uint64_t));
   TRY(send_bytes(header));
   TRY(send_bytes(value));
-  return receive_response_bool();
+  return receive_response_ok_noop_error();
 }
 
 tl::expected<bool, Client::Error>
@@ -220,7 +238,7 @@ Client::remove(std::span<const uint8_t> key)
   msg.push_back(static_cast<uint8_t>(key.size()));
   msg.insert(msg.end(), key.data(), key.size());
   TRY(send_bytes(msg));
-  return receive_response_bool();
+  return receive_response_ok_noop_error();
 }
 
 tl::expected<void, Client::Error>
@@ -328,6 +346,29 @@ Client::receive_error_string()
   return std::string(msg_bytes.begin(), msg_bytes.end());
 }
 
+tl::expected<bool, Client::Error>
+Client::receive_response_exists()
+{
+  TRY_ASSIGN(uint8_t status_byte, receive_u8());
+  auto status = static_cast<Status>(status_byte);
+
+  switch (status) {
+  case Status::ok: {
+    TRY_ASSIGN(uint8_t exists, receive_u8());
+    return exists;
+  }
+
+  case Status::error: {
+    TRY_ASSIGN(auto err_msg, receive_error_string());
+    return tl::unexpected(Error(Failure::error, err_msg));
+  }
+
+  default:
+    return tl::unexpected(
+      Error(Failure::error, FMT("Invalid status code: {}", status_byte)));
+  }
+}
+
 tl::expected<std::optional<util::Bytes>, Client::Error>
 Client::receive_response_get()
 {
@@ -356,7 +397,7 @@ Client::receive_response_get()
 }
 
 tl::expected<bool, Client::Error>
-Client::receive_response_bool()
+Client::receive_response_ok_noop_error()
 {
   TRY_ASSIGN(uint8_t status_byte, receive_u8());
   auto status = static_cast<Status>(status_byte);

diff --git a/src/ccache/storage/remote/client.hpp b/src/ccache/storage/remote/client.hpp
index b63b5e2ada3015e6e2e1dfa7148c3ca8f38cfb7c..70b952ee93699b3568acb12c2984470db5e50184 100644 (file)
--- a/src/ccache/storage/remote/client.hpp
+++ b/src/ccache/storage/remote/client.hpp
@@ -48,8 +48,9 @@ public:
   static constexpr uint8_t k_protocol_version = 0x01;
 
   enum class Capability : uint8_t {
-    get_put_remove = 0x00, // get/put/remove operations
-    info = 0x01,           // info operation
+    get_put_remove = 0x00,
+    info = 0x01,
+    exists = 0x02,
   };
 
   enum class Status : uint8_t {
@@ -95,6 +96,8 @@ public:
   const std::vector<Capability>& capabilities() const;
   bool has_capability(Capability cap) const;
 
+  tl::expected<bool, Error> exists(std::span<const uint8_t> key);
+
   tl::expected<std::optional<util::Bytes>, Error>
   get(std::span<const uint8_t> key);
 
@@ -134,8 +137,9 @@ private:
   tl::expected<void, Error> verify_connected() const;
   static tl::expected<void, Error>
   verify_key_size(std::span<const uint8_t> key);
+  tl::expected<bool, Error> receive_response_exists();
   tl::expected<std::optional<util::Bytes>, Error> receive_response_get();
-  tl::expected<bool, Error> receive_response_bool();
+  tl::expected<bool, Error> receive_response_ok_noop_error();
   tl::expected<void, Error> receive_response_void();
 };
 
@@ -147,6 +151,8 @@ to_string(Client::Capability capability)
     return "get/put/remove";
   case Client::Capability::info:
     return "info";
+  case Client::Capability::exists:
+    return "exists";
   }
   return FMT("{}", static_cast<int>(capability));
 }

diff --git a/src/ccache/storage/remote/helper.cpp b/src/ccache/storage/remote/helper.cpp
index 2c0f0a7dc4a4c7114582d1fbe9b169be88e20a4b..1c017ef9aff3e0a7bc5a0404442f93d3665571e3 100644 (file)
--- a/src/ccache/storage/remote/helper.cpp
+++ b/src/ccache/storage/remote/helper.cpp
@@ -579,8 +579,30 @@ HelperBackend::put(const Hash::Digest& key,
 {
   TRY(ensure_connected());
 
-  Client::PutFlags flags;
-  flags.overwrite = (overwrite == Overwrite::yes);
+  Client::PutFlags flags{.overwrite = true};
+
+  if (overwrite == Overwrite::no) {
+    if (m_client.has_capability(Client::Capability::exists)) {
+      // Prefer asking with exists instead of setting overwrite=false to avoid
+      // having to send the payload to the helper in case the key already
+      // exists.
+      auto result = m_client.exists(key);
+      if (!result) {
+        const auto& error = result.error();
+        LOG("Remote storage exists failed: {}", error.message);
+        auto failure = (error.failure == Client::Failure::timeout)
+                         ? Failure::timeout
+                         : Failure::error;
+        return tl::unexpected(failure);
+      }
+      bool exists = *result;
+      if (exists) {
+        return false;
+      }
+    } else {
+      flags.overwrite = false;
+    }
+  }
 
   auto result = m_client.put(key, value, flags);
   if (!result) {

diff --git a/test/storage/client/main.cpp b/test/storage/client/main.cpp
index fa463bc8ebbfadaac8e36a91466dd675a442625c..e67c00ea21b4811c9d297b328b7891cdf23ba85d 100644 (file)
--- a/test/storage/client/main.cpp
+++ b/test/storage/client/main.cpp
@@ -49,13 +49,15 @@ This is a CLI tool for testing ccache storage helper implementations.
 Commands:
     ping                            check if helper is reachable
     info                            print helper info
+    stop                            tell the helper to stop
+
+    exists KEY                      check if a value exists in storage
     get KEY -o FILE                 get a value and output to file
     get KEY -o -                    get a value and output to stdout
     put [--overwrite] KEY -i FILE   put a value from file
     put [--overwrite] KEY -i -      put a value from stdin
     put [--overwrite] KEY -v VALUE  put a literal value
     remove KEY                      remove a value from storage
-    stop                            tell the helper to stop
 
 Notes:
     KEY must be a hexadecimal string (0-9, a-f, A-F).
@@ -68,6 +70,36 @@ print_usage(FILE* stream, const char* program_name)
   PRINT(stream, USAGE_TEXT, program_name);
 }
 
+tl::expected<int, std::string>
+cmd_exists(Client& client, const std::vector<std::string>& args)
+{
+  if (args.size() != 1) {
+    return tl::unexpected("exists requires exactly 1 argument: KEY");
+  }
+
+  auto key_result = util::parse_base16(args[0]);
+  if (!key_result) {
+    PRINT(stderr, "Error: Invalid hex key: {}\n", key_result.error());
+    return 1;
+  }
+  const auto& key = *key_result;
+
+  auto result = client.exists(key);
+
+  if (!result) {
+    PRINT(stderr, "Error: {}\n", result.error().message);
+    return 1;
+  }
+
+  if (*result) {
+    PRINT(stdout, "yes\n");
+    return 0;
+  } else {
+    PRINT(stderr, "no\n");
+    return 2;
+  }
+}
+
 tl::expected<int, std::string>
 cmd_get(Client& client, const std::vector<std::string>& args)
 {
@@ -264,6 +296,9 @@ handle_command(Client& client,
 
   if (command == "ping") {
     TRY_ASSIGN(result, cmd_ping(args));
+  } else if (command == "exists") {
+    TRY(require_capability(client, Client::Capability::exists));
+    TRY_ASSIGN(result, cmd_exists(client, args));
   } else if (command == "get") {
     TRY(require_capability(client, Client::Capability::get_put_remove));
     TRY_ASSIGN(result, cmd_get(client, args));
@@ -330,4 +365,6 @@ main(int argc, char* argv[])
     PRINT(stderr, "Error: {}\n", result.error());
     return 1;
   }
+
+  return 0;
 }

diff --git a/test/storage/helper/main.cpp b/test/storage/helper/main.cpp
index c3d8fedb09db5b2525cc4b448b40dc125dd1266d..1e67a22149da032be413ee10bd1c7c49f41c278d 100644 (file)
--- a/test/storage/helper/main.cpp
+++ b/test/storage/helper/main.cpp
@@ -59,6 +59,7 @@ namespace fs = util::filesystem;
 constexpr uint8_t PROTOCOL_VERSION = 0x01;
 constexpr uint8_t CAP_GET_PUT_REMOVE = 0x00;
 constexpr uint8_t CAP_INFO = 0x01;
+constexpr uint8_t CAP_EXISTS = 0x02;
 
 constexpr uint8_t STATUS_OK = 0x00;
 constexpr uint8_t STATUS_NOOP = 0x01;
@@ -69,9 +70,18 @@ constexpr uint8_t REQ_PUT = 0x01;
 constexpr uint8_t REQ_REMOVE = 0x02;
 constexpr uint8_t REQ_STOP = 0x03;
 constexpr uint8_t REQ_INFO = 0x04;
+constexpr uint8_t REQ_EXISTS = 0x05;
 
 constexpr uint8_t PUT_FLAG_OVERWRITE = 0x01;
 
+constexpr uint8_t GREETING[] = {
+  PROTOCOL_VERSION,
+  3,
+  CAP_GET_PUT_REMOVE,
+  CAP_INFO,
+  CAP_EXISTS,
+};
+
 #ifdef _WIN32
 using ConnHandle = HANDLE;
 #else
@@ -125,6 +135,7 @@ private:
   bool recv_exact(ConnHandle conn, uint8_t* buf, size_t count);
   void send_data(ConnHandle conn, const uint8_t* data, size_t len);
   void send_error(ConnHandle conn, const char* message);
+  void handle_exists(ConnHandle conn);
   void handle_get(ConnHandle conn);
   void handle_info(ConnHandle conn);
   void handle_put(ConnHandle conn);
@@ -220,6 +231,36 @@ IpcServer::send_error(ConnHandle conn, const char* message)
   send_data(conn, response.data(), response.size());
 }
 
+void
+IpcServer::handle_exists(ConnHandle conn)
+{
+  uint8_t key_len;
+  if (!recv_exact(conn, &key_len, 1)) {
+    return;
+  }
+
+  std::string key;
+  if (key_len > 0) {
+    key.resize(key_len);
+    if (!recv_exact(conn, reinterpret_cast<uint8_t*>(key.data()), key_len)) {
+      return;
+    }
+  }
+
+  log_msg(FMT("EXISTS: key_len={}", key_len));
+
+  auto it = m_storage.find(key);
+  if (it != m_storage.end()) {
+    static uint8_t response[] = {STATUS_OK, 0x01};
+    send_data(conn, response, sizeof(response));
+    log_msg("  -> exists");
+  } else {
+    static uint8_t response[] = {STATUS_OK, 0x00};
+    send_data(conn, response, sizeof(response));
+    log_msg("  -> does not exist");
+  }
+}
+
 void
 IpcServer::handle_get(ConnHandle conn)
 {
@@ -371,8 +412,7 @@ IpcServer::handle_stop(ConnHandle conn)
 void
 IpcServer::handle_client(ConnHandle conn)
 {
-  uint8_t greeting[] = {PROTOCOL_VERSION, 2, CAP_GET_PUT_REMOVE, CAP_INFO};
-  send_data(conn, greeting, sizeof(greeting));
+  send_data(conn, GREETING, sizeof(GREETING));
 
   while (true) {
     uint8_t request_type;
@@ -383,6 +423,10 @@ IpcServer::handle_client(ConnHandle conn)
     update_activity();
 
     switch (request_type) {
+    case REQ_EXISTS:
+      handle_exists(conn);
+      break;
+
     case REQ_GET:
       handle_get(conn);
       break;