From: Joel Rosdahl <joel@rosdahl.net>
Date: Sat, 7 Aug 2021 13:43:24 +0000 (+0200)
Subject: feat(file-storage): Add support for flat layout
X-Git-Tag: v4.4~45
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=13c0ef12df968302f6607bfc74699025490ce61e;p=thirdparty%2Fccache.git

feat(file-storage): Add support for flat layout
---

diff --git a/doc/MANUAL.adoc b/doc/MANUAL.adoc
index b61c24263..cf804641e 100644
--- a/doc/MANUAL.adoc
+++ b/doc/MANUAL.adoc
@@ -936,6 +936,14 @@ Examples:
 
 Optional attributes:
 
+* *layout*: How to store file under the cache directory. Available values:
++
+--
+* *flat*: Store all files directly under the cache directory.
+* *subdirs*: Store files in 256 subdirectories of the cache directory.
+--
++
+The default is *subdirs*.
 * *umask*: This attribute (an octal integer) overrides the umask to use for
   files and directories in the cache directory.
 * *update-mtime*: If *true*, update the modification time (mtime) of cache
diff --git a/src/storage/secondary/FileStorage.cpp b/src/storage/secondary/FileStorage.cpp
index f5273ff85..7312c62a4 100644
--- a/src/storage/secondary/FileStorage.cpp
+++ b/src/storage/secondary/FileStorage.cpp
@@ -54,9 +54,12 @@ public:
   nonstd::expected<bool, Failure> remove(const Digest& key) override;
 
 private:
+  enum class Layout { flat, subdirs };
+
   const std::string m_dir;
   nonstd::optional<mode_t> m_umask;
   bool m_update_mtime = false;
+  Layout m_layout = Layout::subdirs;
 
   std::string get_entry_path(const Digest& key) const;
 };
@@ -73,7 +76,15 @@ FileStorageBackend::FileStorageBackend(const Params& params)
   }
 
   for (const auto& attr : params.attributes) {
-    if (attr.key == "umask") {
+    if (attr.key == "layout") {
+      if (attr.value == "flat") {
+        m_layout = Layout::flat;
+      } else if (attr.value == "subdirs") {
+        m_layout = Layout::subdirs;
+      } else {
+        LOG("Unknown layout: {}", attr.value);
+      }
+    } else if (attr.key == "umask") {
       m_umask =
         util::value_or_throw<core::Fatal>(util::parse_umask(attr.value));
     } else if (attr.key == "update-mtime") {
@@ -156,9 +167,19 @@ FileStorageBackend::remove(const Digest& key)
 std::string
 FileStorageBackend::get_entry_path(const Digest& key) const
 {
-  const auto key_string = key.to_string();
-  const uint8_t digits = 2;
-  return FMT("{}/{:.{}}/{}", m_dir, key_string, digits, &key_string[digits]);
+  switch (m_layout) {
+  case Layout::flat:
+    return FMT("{}/{}", m_dir, key.to_string());
+
+  case Layout::subdirs: {
+    const auto key_str = key.to_string();
+    const uint8_t digits = 2;
+    ASSERT(key_str.length() > digits);
+    return FMT("{}/{:.{}}/{}", m_dir, key_str, digits, &key_str[digits]);
+  }
+  }
+
+  ASSERT(false);
 }
 
 } // namespace
diff --git a/test/suites/secondary_file.bash b/test/suites/secondary_file.bash
index adc455b5e..6fce2966e 100644
--- a/test/suites/secondary_file.bash
+++ b/test/suites/secondary_file.bash
@@ -7,13 +7,49 @@ SUITE_secondary_file_SETUP() {
 
 SUITE_secondary_file() {
     # -------------------------------------------------------------------------
-    TEST "Base case"
+    TEST "Subdirs layout"
 
     $CCACHE_COMPILE -c test.c
     expect_stat 'cache hit (direct)' 0
     expect_stat 'cache miss' 1
     expect_stat 'files in cache' 2
     expect_exists secondary/CACHEDIR.TAG
+    subdirs=$(find secondary -type d | wc -l)
+    if [ "${subdirs}" -lt 2 ]; then # "secondary" itself counts as one
+        test_failed "Expected subdirectories in secondary"
+    fi
+    expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
+
+    $CCACHE_COMPILE -c test.c
+    expect_stat 'cache hit (direct)' 1
+    expect_stat 'cache miss' 1
+    expect_stat 'files in cache' 2
+    expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
+
+    $CCACHE -C >/dev/null
+    expect_stat 'files in cache' 0
+    expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
+
+    $CCACHE_COMPILE -c test.c
+    expect_stat 'cache hit (direct)' 2
+    expect_stat 'cache miss' 1
+    expect_stat 'files in cache' 2 # fetched from secondary
+    expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
+
+    # -------------------------------------------------------------------------
+    TEST "Flat layout"
+
+    CCACHE_SECONDARY_STORAGE+="|layout=flat"
+
+    $CCACHE_COMPILE -c test.c
+    expect_stat 'cache hit (direct)' 0
+    expect_stat 'cache miss' 1
+    expect_stat 'files in cache' 2
+    expect_exists secondary/CACHEDIR.TAG
+    subdirs=$(find secondary -type d | wc -l)
+    if [ "${subdirs}" -ne 1 ]; then # "secondary" itself counts as one
+        test_failed "Expected no subdirectories in secondary"
+    fi
     expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
 
     $CCACHE_COMPILE -c test.c