feat(http-storage): Add support for the Bazel HTTP caching protocol

diff --git a/doc/MANUAL.adoc b/doc/MANUAL.adoc
index 1bcb8b80c02fc1b8dfe39c95214b477e743fab7a..05f337dd6c26fedae7116b66fb90fc9617e68f5d 100644 (file)
--- a/doc/MANUAL.adoc
+++ b/doc/MANUAL.adoc
@@ -938,12 +938,30 @@ Note that ccache will not perform any cleanup of the HTTP storage.
 
 Examples:
 
-* `+http://localhost:8080/+`
-* `+http://someusername:p4ssw0rd@example.com/cache+`
+* `+http://localhost+`
+* `+http://someusername:p4ssw0rd@example.com/cache/+`
+* `+http://localhost:8080|layout=bazel|connect-timeout=50+`
 
 Optional attributes:
 
 * *connect-timeout*: Timeout (in ms) for network connection. The default is 100.
+* *layout*: How to map key names to the path part of the URL. Available values:
++
+--
+* *bazel*: Store values in a format compatible with the Bazel HTTP caching
+   protocol. More specifically, the entries will be stored as 64 hex digits
+   under the `/ac/` part of the cache.
++
+NOTE: You may have to disable verification of action cache values in the server
+for this to work since ccache entries are not valid action result metadata
+values.
+* *standard*: Append the first two digits of ccache's standard text
+  representation of a hash to the URL (with a leading slash if needed), followed
+  by a slash and the rest of the key. This divides the entries into 256 buckets
+  since the first digits are in hex format.
+--
++
+The default is *standard*.
 * *operation-timeout*: Timeout (in ms) for HTTP requests. The default is 10000.
 
 Known issues and limitations:

diff --git a/src/storage/secondary/HttpStorage.cpp b/src/storage/secondary/HttpStorage.cpp
index 1e1a58df0bfe1f0e086b68d247acca33429ab6c0..2867e1b83681438bcac8063747a007dd19eeaed0 100644 (file)
--- a/src/storage/secondary/HttpStorage.cpp
+++ b/src/storage/secondary/HttpStorage.cpp
@@ -50,8 +50,11 @@ public:
   nonstd::expected<bool, Failure> remove(const Digest& key) override;
 
 private:
+  enum class Layout { bazel, standard };
+
   const std::string m_url_path;
   httplib::Client m_http_client;
+  Layout m_layout = Layout::standard;
 
   std::string get_entry_path(const Digest& key) const;
 };
@@ -115,6 +118,14 @@ HttpStorageBackend::HttpStorageBackend(const Params& params)
   for (const auto& attr : params.attributes) {
     if (attr.key == "connect-timeout") {
       connect_timeout = parse_timeout_attribute(attr.value);
+    } else if (attr.key == "layout") {
+      if (attr.value == "bazel") {
+        m_layout = Layout::bazel;
+      } else if (attr.value == "standard") {
+        m_layout = Layout::standard;
+      } else {
+        LOG("Unknown layout: {}", attr.value);
+      }
     } else if (attr.key == "operation-timeout") {
       operation_timeout = parse_timeout_attribute(attr.value);
     } else if (!is_framework_attribute(attr.key)) {
@@ -225,7 +236,21 @@ HttpStorageBackend::remove(const Digest& key)
 std::string
 HttpStorageBackend::get_entry_path(const Digest& key) const
 {
-  return m_url_path + key.to_string();
+  switch (m_layout) {
+  case Layout::bazel: {
+    // Mimic hex representation of a SHA256 hash value.
+    const auto sha256_hex_size = 64;
+    static_assert(Digest::size() == 20, "Update below if digest size changes");
+    std::string hex_digits = Util::format_base16(key.bytes(), key.size());
+    hex_digits.append(hex_digits.data(), sha256_hex_size - hex_digits.size());
+    return FMT("{}ac/{}", m_url_path, hex_digits);
+  }
+
+  case Layout::standard:
+    return m_url_path + key.to_string();
+  }
+
+  ASSERT(false);
 }
 
 } // namespace

diff --git a/test/suites/secondary_http.bash b/test/suites/secondary_http.bash
index f9974f41779c62dd4bd2b6d9c6c6878d9599bc60..ca71f43ab5c2322fd936f943ac216f67fc5166cf 100644 (file)
--- a/test/suites/secondary_http.bash
+++ b/test/suites/secondary_http.bash
@@ -136,4 +136,38 @@ SUITE_secondary_http() {
         expect_stat 'files in cache' 2 # fetched from secondary
         expect_file_count 2 '*' secondary # result + manifest
     fi
+
+    # -------------------------------------------------------------------------
+    TEST "Bazel layout"
+
+    start_http_server 12780 secondary
+    mkdir secondary/ac
+    export CCACHE_SECONDARY_STORAGE="http://localhost:12780|layout=bazel"
+
+    $CCACHE_COMPILE -c test.c
+    expect_stat 'cache hit (direct)' 0
+    expect_stat 'cache miss' 1
+    expect_stat 'files in cache' 2
+    find secondary -ls
+    expect_file_count 2 '*' secondary/ac # result + manifest
+    if [ "$(ls secondary/ac | grep -Ec '^[0-9a-f]{64}$')" -ne 2 ]; then
+        test_failed "Bazel layout filenames not as expected"
+    fi
+
+    $CCACHE_COMPILE -c test.c
+    expect_stat 'cache hit (direct)' 1
+    expect_stat 'cache miss' 1
+    expect_stat 'files in cache' 2
+    expect_file_count 2 '*' secondary/ac # result + manifest
+
+    $CCACHE -C >/dev/null
+    expect_stat 'files in cache' 0
+    expect_file_count 2 '*' secondary/ac # result + manifest
+
+    $CCACHE_COMPILE -c test.c
+    expect_stat 'cache hit (direct)' 2
+    expect_stat 'cache miss' 1
+    expect_stat 'files in cache' 2 # fetched from secondary
+    expect_file_count 2 '*' secondary/ac # result + manifest
+
 }