From: Joel Rosdahl Date: Thu, 10 Nov 2022 09:07:33 +0000 (+0100) Subject: refactor: Improve FileRecompressor to take a stat X-Git-Tag: v4.8~89 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=5d2f3c702710acd9fc9f71b2a0c3545b3ad5f841;p=thirdparty%2Fccache.git refactor: Improve FileRecompressor to take a stat This avoids extra stats in some scenarios. --- diff --git a/src/core/FileRecompressor.cpp b/src/core/FileRecompressor.cpp index 131a85f95..636d6761f 100644 --- a/src/core/FileRecompressor.cpp +++ b/src/core/FileRecompressor.cpp @@ -27,23 +27,23 @@ namespace core { -int64_t -FileRecompressor::recompress(const std::string& cache_file, +Stat +FileRecompressor::recompress(const Stat& stat, std::optional level, KeepAtime keep_atime) { - core::CacheEntry::Header header(cache_file); + core::CacheEntry::Header header(stat.path()); const int8_t wanted_level = level ? (*level == 0 ? core::CacheEntry::default_compression_level : *level) : 0; - const auto old_stat = Stat::lstat(cache_file, Stat::OnError::log); - Stat new_stat(old_stat); + + std::optional new_stat; if (header.compression_level != wanted_level) { const auto cache_file_data = util::value_or_throw( - util::read_file(cache_file), - FMT("Failed to read {}: ", cache_file)); + util::read_file(stat.path()), + FMT("Failed to read {}: ", stat.path())); core::CacheEntry cache_entry(cache_file_data); cache_entry.verify_checksum(); @@ -52,23 +52,23 @@ FileRecompressor::recompress(const std::string& cache_file, level ? core::CompressionType::zstd : core::CompressionType::none; header.compression_level = wanted_level; - AtomicFile new_cache_file(cache_file, AtomicFile::Mode::binary); + AtomicFile new_cache_file(stat.path(), AtomicFile::Mode::binary); new_cache_file.write( core::CacheEntry::serialize(header, cache_entry.payload())); new_cache_file.commit(); - new_stat = Stat::lstat(cache_file, Stat::OnError::log); + new_stat = Stat::lstat(stat.path(), Stat::OnError::log); } // Restore mtime/atime to keep cache LRU cleanup working as expected: if (keep_atime == KeepAtime::yes || new_stat) { - util::set_timestamps(cache_file, old_stat.mtime(), old_stat.atime()); + util::set_timestamps(stat.path(), stat.mtime(), stat.atime()); } m_content_size += header.entry_size; - m_old_size += old_stat.size_on_disk(); - m_new_size += new_stat.size_on_disk(); + m_old_size += stat.size_on_disk(); + m_new_size += (new_stat ? *new_stat : stat).size_on_disk(); - return Util::size_change_kibibyte(old_stat, new_stat); + return new_stat ? *new_stat : stat; } uint64_t diff --git a/src/core/FileRecompressor.hpp b/src/core/FileRecompressor.hpp index b2fdb131b..84ce28246 100644 --- a/src/core/FileRecompressor.hpp +++ b/src/core/FileRecompressor.hpp @@ -18,6 +18,8 @@ #pragma once +#include + #include #include #include @@ -33,10 +35,10 @@ public: FileRecompressor() = default; - // Returns on-disk size change in KiB. - int64_t recompress(const std::string& cache_file, - const std::optional level, - KeepAtime keep_atime); + // Returns stat after recompression. + Stat recompress(const Stat& stat, + std::optional level, + KeepAtime keep_atime); uint64_t content_size() const; uint64_t old_size() const; diff --git a/src/core/mainoptions.cpp b/src/core/mainoptions.cpp index 7b3e639ad..9ebe5cee9 100644 --- a/src/core/mainoptions.cpp +++ b/src/core/mainoptions.cpp @@ -273,19 +273,14 @@ trim_dir(const std::string& dir, std::optional> recompress_level, uint32_t recompress_threads) { - struct File - { - std::string path; - Stat stat; - }; - std::vector files; + std::vector files; uint64_t initial_size = 0; Util::traverse(dir, [&](const std::string& path, const bool is_dir) { if (is_dir || TemporaryFile::is_tmp_file(path)) { return; } - const auto stat = Stat::lstat(path); + auto stat = Stat::lstat(path); if (!stat) { // Probably some race, ignore. return; @@ -296,12 +291,11 @@ trim_dir(const std::string& dir, throw Fatal( FMT("this looks like a local cache directory (found {})", path)); } - files.push_back({path, stat}); + files.emplace_back(std::move(stat)); }); std::sort(files.begin(), files.end(), [&](const auto& f1, const auto& f2) { - return trim_lru_mtime ? f1.stat.mtime() < f2.stat.mtime() - : f1.stat.atime() < f2.stat.atime(); + return trim_lru_mtime ? f1.mtime() < f2.mtime() : f1.atime() < f2.atime(); }); int64_t recompression_diff = 0; @@ -313,15 +307,15 @@ trim_dir(const std::string& dir, core::FileRecompressor recompressor; std::atomic incompressible_size = 0; - for (const auto& file : files) { + for (auto& file : files) { thread_pool.enqueue([&] { try { - recompressor.recompress(file.path, - *recompress_level, - core::FileRecompressor::KeepAtime::yes); + auto new_stat = recompressor.recompress( + file, *recompress_level, core::FileRecompressor::KeepAtime::yes); + file = std::move(new_stat); // Remember new size, if any. } catch (core::Error&) { // Ignore for now. - incompressible_size += file.stat.size_on_disk(); + incompressible_size += file.size_on_disk(); } }); } @@ -345,9 +339,9 @@ trim_dir(const std::string& dir, if (final_size <= trim_max_size) { break; } - if (Util::unlink_tmp(file.path)) { + if (Util::unlink_tmp(file.path())) { ++removed_files; - final_size -= file.stat.size_on_disk(); + final_size -= file.size_on_disk(); } } diff --git a/src/storage/local/LocalStorage_compress.cpp b/src/storage/local/LocalStorage_compress.cpp index 6fbe0eba6..fd4d258aa 100644 --- a/src/storage/local/LocalStorage_compress.cpp +++ b/src/storage/local/LocalStorage_compress.cpp @@ -109,12 +109,16 @@ LocalStorage::recompress(const std::optional level, thread_pool.enqueue( [&recompressor, &incompressible_size, level, stats_file, file] { try { - int64_t size_change_kibibyte = recompressor.recompress( - file.path(), level, core::FileRecompressor::KeepAtime::no); - StatsFile(stats_file).update([=](auto& cs) { - cs.increment(core::Statistic::cache_size_kibibyte, - size_change_kibibyte); - }); + Stat new_stat = recompressor.recompress( + file, level, core::FileRecompressor::KeepAtime::no); + auto size_change_kibibyte = + Util::size_change_kibibyte(file, new_stat); + if (size_change_kibibyte != 0) { + StatsFile(stats_file).update([=](auto& cs) { + cs.increment(core::Statistic::cache_size_kibibyte, + size_change_kibibyte); + }); + } } catch (core::Error&) { // Ignore for now. incompressible_size += file.size_on_disk();