From: Joel Rosdahl <joel@rosdahl.net>
Date: Sat, 7 Aug 2021 12:44:21 +0000 (+0200)
Subject: feat(http-storage): Add support for subdirs layout, making it default
X-Git-Tag: v4.4~46
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=21f1afba517037f90c029365378c6f5606e51f05;p=thirdparty%2Fccache.git

feat(http-storage): Add support for subdirs layout, making it default
---

diff --git a/doc/MANUAL.adoc b/doc/MANUAL.adoc
index 6bc4651b7..b61c24263 100644
--- a/doc/MANUAL.adoc
+++ b/doc/MANUAL.adoc
@@ -975,13 +975,14 @@ Optional attributes:
 NOTE: You may have to disable verification of action cache values in the server
 for this to work since ccache entries are not valid action result metadata
 values.
-* *standard*: Append the first two digits of ccache's standard text
-  representation of a hash to the URL (with a leading slash if needed), followed
-  by a slash and the rest of the key. This divides the entries into 256 buckets
-  since the first digits are in hex format.
+* *flat*: Append the key directly to the path part of the URL (with a leading
+   slash if needed).
+* *subdirs*: Append the first two characters of the key to the URL (with a
+  leading slash if needed), followed by a slash and the rest of the key. This
+  divides the entries into 256 buckets.
 --
 +
-The default is *standard*.
+The default is *subdirs*.
 * *operation-timeout*: Timeout (in ms) for HTTP requests. The default is 10000.
 
 
diff --git a/src/storage/secondary/HttpStorage.cpp b/src/storage/secondary/HttpStorage.cpp
index 3aa5e58e8..6b3d00268 100644
--- a/src/storage/secondary/HttpStorage.cpp
+++ b/src/storage/secondary/HttpStorage.cpp
@@ -50,11 +50,11 @@ public:
   nonstd::expected<bool, Failure> remove(const Digest& key) override;
 
 private:
-  enum class Layout { bazel, standard };
+  enum class Layout { bazel, flat, subdirs };
 
   const std::string m_url_path;
   httplib::Client m_http_client;
-  Layout m_layout = Layout::standard;
+  Layout m_layout = Layout::subdirs;
 
   std::string get_entry_path(const Digest& key) const;
 };
@@ -121,8 +121,10 @@ HttpStorageBackend::HttpStorageBackend(const Params& params)
     } else if (attr.key == "layout") {
       if (attr.value == "bazel") {
         m_layout = Layout::bazel;
-      } else if (attr.value == "standard") {
-        m_layout = Layout::standard;
+      } else if (attr.value == "flat") {
+        m_layout = Layout::flat;
+      } else if (attr.value == "subdirs") {
+        m_layout = Layout::subdirs;
       } else {
         LOG("Unknown layout: {}", attr.value);
       }
@@ -247,8 +249,15 @@ HttpStorageBackend::get_entry_path(const Digest& key) const
     return FMT("{}ac/{}", m_url_path, hex_digits);
   }
 
-  case Layout::standard:
+  case Layout::flat:
     return m_url_path + key.to_string();
+
+  case Layout::subdirs: {
+    const auto key_str = key.to_string();
+    const uint8_t digits = 2;
+    ASSERT(key_str.length() > digits);
+    return FMT("{}/{:.{}}/{}", m_url_path, key_str, digits, &key_str[digits]);
+  }
   }
 
   ASSERT(false);
diff --git a/test/suites/secondary_http.bash b/test/suites/secondary_http.bash
index fb258ffaf..f4377980a 100644
--- a/test/suites/secondary_http.bash
+++ b/test/suites/secondary_http.bash
@@ -40,7 +40,7 @@ SUITE_secondary_http_SETUP() {
 
 SUITE_secondary_http() {
     # -------------------------------------------------------------------------
-    TEST "Base case"
+    TEST "Subdirs layout"
 
     start_http_server 12780 secondary
     export CCACHE_SECONDARY_STORAGE="http://localhost:12780"
@@ -50,6 +50,10 @@ SUITE_secondary_http() {
     expect_stat 'cache miss' 1
     expect_stat 'files in cache' 2
     expect_file_count 2 '*' secondary # result + manifest
+    subdirs=$(find secondary -type d | wc -l)
+    if [ "${subdirs}" -lt 2 ]; then # "secondary" itself counts as one
+        test_failed "Expected subdirectories in secondary"
+    fi
 
     $CCACHE_COMPILE -c test.c
     expect_stat 'cache hit (direct)' 1
@@ -67,6 +71,37 @@ SUITE_secondary_http() {
     expect_stat 'files in cache' 2 # fetched from secondary
     expect_file_count 2 '*' secondary # result + manifest
 
+    # -------------------------------------------------------------------------
+    TEST "Flat layout"
+
+    start_http_server 12780 secondary
+    export CCACHE_SECONDARY_STORAGE="http://localhost:12780|layout=flat"
+
+    $CCACHE_COMPILE -c test.c
+    expect_stat 'cache hit (direct)' 0
+    expect_stat 'cache miss' 1
+    expect_stat 'files in cache' 2
+    expect_file_count 2 '*' secondary # result + manifest
+    subdirs=$(find secondary -type d | wc -l)
+    if [ "${subdirs}" -ne 1 ]; then # "secondary" itself counts as one
+        test_failed "Expected no subdirectories in secondary"
+    fi
+
+    $CCACHE_COMPILE -c test.c
+    expect_stat 'cache hit (direct)' 1
+    expect_stat 'cache miss' 1
+    expect_stat 'files in cache' 2
+    expect_file_count 2 '*' secondary # result + manifest
+
+    $CCACHE -C >/dev/null
+    expect_stat 'files in cache' 0
+    expect_file_count 2 '*' secondary # result + manifest
+
+    $CCACHE_COMPILE -c test.c
+    expect_stat 'cache hit (direct)' 2
+    expect_stat 'cache miss' 1
+    expect_stat 'files in cache' 2 # fetched from secondary
+    expect_file_count 2 '*' secondary # result + manifest
 
     # -------------------------------------------------------------------------
     TEST "Bazel layout"