fix: Let DirEntry::size_on_disk return likely size on disk

st_blocks in struct stat cannot be trusted for some filesystems that do
transparent compression or deduplication. For instance, ZFS can adjust
the block count some time in the future when compression is finished.

The only reasonable fix for this seems to be to just guess how much
space the file will take using st_size.

Fixes #1332.

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index daed99916c29c947e43b2b8e0af473794b8a82cf..c74e28f2e00e632154f68c54666146f02a7fe3f6 100644 (file)
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -138,7 +138,6 @@ jobs:
       - name: Build and test
         run: ci/build
         env:
-          ENABLE_CACHE_CLEANUP_TESTS: true
           CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI
 
       - name: Collect testdir from failed tests
@@ -206,7 +205,6 @@ jobs:
         continue-on-error: ${{ matrix.config.allow_test_failures == true &&
           steps.build-and-test.outputs.exit_status == 8 }}
         env:
-          ENABLE_CACHE_CLEANUP_TESTS: 1
           CMAKE_GENERATOR: Ninja
           CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI
           TEST_CC: gcc
@@ -254,7 +252,6 @@ jobs:
             os: ubuntu-20.04
             CC: gcc
             CXX: g++
-            ENABLE_CACHE_CLEANUP_TESTS: 1
             BUILDDIR: .
             CCACHE_LOC: .
             CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=Debug -DENABLE_TRACING=1
@@ -268,7 +265,6 @@ jobs:
             CXXFLAGS: -m32 -g -O2
             LDFLAGS: -m32
             CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON
-            ENABLE_CACHE_CLEANUP_TESTS: 1
             apt_get: elfutils gcc-multilib g++-multilib lib32stdc++-10-dev
 
           - name: Linux GCC CUDA
@@ -276,7 +272,6 @@ jobs:
             CC: gcc
             CXX: g++
             CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON
-            ENABLE_CACHE_CLEANUP_TESTS: 1
             CUDA: 11.7.0-1
             apt_get: elfutils libzstd-dev
 
@@ -292,7 +287,6 @@ jobs:
             os: ubuntu-20.04
             CC: x86_64-w64-mingw32-gcc-posix
             CXX: x86_64-w64-mingw32-g++-posix
-            ENABLE_CACHE_CLEANUP_TESTS: 1
             CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DCMAKE_SYSTEM_NAME=Windows -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON
             RUN_TESTS: unittest-in-wine
             apt_get: elfutils mingw-w64 wine
@@ -302,7 +296,6 @@ jobs:
             msvc_arch: x64_x86
             CC: cl
             CXX: cl
-            ENABLE_CACHE_CLEANUP_TESTS: 1
             CMAKE_GENERATOR: Ninja
             CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON
             TEST_CC: clang -target i686-pc-windows-msvc
@@ -313,7 +306,6 @@ jobs:
             allow_test_failures: true  # For now, don't fail the build on failure
             CC: cl
             CXX: cl
-            ENABLE_CACHE_CLEANUP_TESTS: 1
             CMAKE_GENERATOR: Visual Studio 16 2019
             CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON -A Win32
             TEST_CC: clang -target i686-pc-windows-msvc
@@ -323,7 +315,6 @@ jobs:
             msvc_arch: x64
             CC: cl
             CXX: cl
-            ENABLE_CACHE_CLEANUP_TESTS: 1
             CMAKE_GENERATOR: Ninja
             CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON
             TEST_CC: clang -target x86_64-pc-windows-msvc
@@ -334,7 +325,6 @@ jobs:
             allow_test_failures: true  # For now, don't fail the build on failure
             CC: cl
             CXX: cl
-            ENABLE_CACHE_CLEANUP_TESTS: 1
             CMAKE_GENERATOR: Visual Studio 16 2019
             CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON -A x64
             TEST_CC: clang -target x86_64-pc-windows-msvc
@@ -345,7 +335,6 @@ jobs:
             allow_test_failures: true  # For now, don't fail the build on failure
             CC: cl
             CXX: cl
-            ENABLE_CACHE_CLEANUP_TESTS: 1
             CMAKE_GENERATOR: Ninja
             CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON
             TEST_CC: clang -target i686-pc-windows-msvc
@@ -356,7 +345,6 @@ jobs:
             allow_test_failures: true  # For now, don't fail the build on failure
             CC: cl
             CXX: cl
-            ENABLE_CACHE_CLEANUP_TESTS: 1
             CMAKE_GENERATOR: Visual Studio 17 2022
             CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON -A Win32
             TEST_CC: clang -target i686-pc-windows-msvc
@@ -367,7 +355,6 @@ jobs:
             allow_test_failures: true  # For now, don't fail the build on failure
             CC: cl
             CXX: cl
-            ENABLE_CACHE_CLEANUP_TESTS: 1
             CMAKE_GENERATOR: Ninja
             CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON
             TEST_CC: clang -target x86_64-pc-windows-msvc
@@ -378,7 +365,6 @@ jobs:
             allow_test_failures: true  # For now, don't fail the build on failure
             CC: cl
             CXX: cl
-            ENABLE_CACHE_CLEANUP_TESTS: 1
             CMAKE_GENERATOR: Visual Studio 17 2022
             CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON -A x64
             TEST_CC: clang -target x86_64-pc-windows-msvc
@@ -387,7 +373,6 @@ jobs:
             os: ubuntu-20.04
             CC: clang
             CXX: clang++
-            ENABLE_CACHE_CLEANUP_TESTS: 1
             CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DENABLE_SANITIZER_ADDRESS=ON -DENABLE_SANITIZER_UNDEFINED_BEHAVIOR=ON
             ASAN_OPTIONS: detect_leaks=0
             apt_get: elfutils libzstd-dev pkg-config libhiredis-dev
@@ -396,7 +381,6 @@ jobs:
             os: ubuntu-20.04
             CC: clang
             CXX: clang++
-            ENABLE_CACHE_CLEANUP_TESTS: 1
             CMAKE_PREFIX: scan-build
             RUN_TESTS: none
             apt_get: libzstd-dev pkg-config libhiredis-dev
@@ -485,7 +469,6 @@ jobs:
           CMAKE_PARAMS: ${{ matrix.config.CMAKE_PARAMS }}
           CXX: ${{ matrix.config.CXX }}
           CXXFLAGS: ${{ matrix.config.CXXFLAGS }}
-          ENABLE_CACHE_CLEANUP_TESTS: ${{ matrix.config.ENABLE_CACHE_CLEANUP_TESTS }}
           EXTRA_CMAKE_BUILD_FLAGS: ${{ matrix.config.EXTRA_CMAKE_BUILD_FLAGS }}
           LDFLAGS: ${{ matrix.config.LDFLAGS }}
           RUN_TESTS: ${{ matrix.config.RUN_TESTS }}

diff --git a/src/util/DirEntry.cpp b/src/util/DirEntry.cpp
index 44007e5374a6d4f8fac37cf0ff056f0e73c1cae7..b031181c5e903fb5c234c8825049720662748db3 100644 (file)
--- a/src/util/DirEntry.cpp
+++ b/src/util/DirEntry.cpp
@@ -239,11 +239,7 @@ namespace util {
 uint64_t
 DirEntry::size_on_disk() const
 {
-#ifdef _WIN32
   return util::likely_size_on_disk(size());
-#else
-  return do_stat().st_blocks * 512;
-#endif
 }
 
 const DirEntry::stat_t&

diff --git a/test/suites/cleanup.bash b/test/suites/cleanup.bash
index 53189f268f3c2b83fefdad74ddb922cf84525c7e..9f2dd20a6c8f3feb524d5b2cc869fec11a15a953 100644 (file)
--- a/test/suites/cleanup.bash
+++ b/test/suites/cleanup.bash
@@ -1,15 +1,3 @@
-SUITE_cleanup_PROBE() {
-    # NOTE: This test suite is known to fail on filesystems that have unusual
-    # block sizes, including ecryptfs. The workaround is to place the test
-    # directory elsewhere:
-    #
-    #     cd /tmp
-    #     CCACHE=$DIR/ccache $DIR/test.sh
-    if [ -z "$ENABLE_CACHE_CLEANUP_TESTS" ]; then
-        echo "ENABLE_CACHE_CLEANUP_TESTS is not set"
-    fi
-}
-
 SUITE_cleanup_SETUP() {
     mkdir -p $CCACHE_DIR/0/0
     printf 'A%.0s' {1..4017} >"$CCACHE_DIR/0/0/result0R"

diff --git a/test/suites/trim_dir.bash b/test/suites/trim_dir.bash
index 3297930d4dab3987ecffe553784e4915fc7cdc52..e3fb97d068f77417de3e9f9dff6bc09e1d17c0bc 100644 (file)
--- a/test/suites/trim_dir.bash
+++ b/test/suites/trim_dir.bash
@@ -1,9 +1,3 @@
-SUITE_trim_dir_PROBE() {
-    if [ -z "$ENABLE_CACHE_CLEANUP_TESTS" ]; then
-        echo "cleanup tests disabled"
-    fi
-}
-
 SUITE_trim_dir() {
     # -------------------------------------------------------------------------
     TEST "Trim remote cache directory"

diff --git a/unittest/test_util_DirEntry.cpp b/unittest/test_util_DirEntry.cpp
index ebd349d6bf64755542cc054a04018b4a8883e029..1c98263d09ea233357ac91112384a5ed4e5f201f 100644 (file)
--- a/unittest/test_util_DirEntry.cpp
+++ b/unittest/test_util_DirEntry.cpp
@@ -290,7 +290,7 @@ TEST_CASE("Return values when file exists")
   CHECK(de.device() == st.st_dev);
   CHECK(de.inode() == st.st_ino);
   CHECK(de.mode() == st.st_mode);
-  CHECK(de.size_on_disk() == st.st_blocks * 512);
+  CHECK(de.size_on_disk() == util::likely_size_on_disk(st.st_size));
 
 #  ifdef HAVE_STRUCT_STAT_ST_CTIM
   CHECK(de.ctime().sec() == st.st_ctim.tv_sec);