From: Joel Rosdahl Date: Wed, 28 Aug 2019 20:41:23 +0000 (+0200) Subject: Add util::get_level_1_files function X-Git-Tag: v4.0~804 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=203272491a5cc9809f27d2bc7f5f92ce16b26afc;p=thirdparty%2Fccache.git Add util::get_level_1_files function --- diff --git a/dev.mk.in b/dev.mk.in index 504f1be2b..2561fe5bb 100644 --- a/dev.mk.in +++ b/dev.mk.in @@ -36,6 +36,7 @@ built_dist_files = $(generated_sources) $(generated_docs) non_third_party_headers = \ src/AtomicFile.hpp \ + src/CacheFile.hpp \ src/Config.hpp \ src/Error.hpp \ src/ProgressBar.hpp \ diff --git a/src/CacheFile.hpp b/src/CacheFile.hpp new file mode 100644 index 000000000..4a6c8fcdb --- /dev/null +++ b/src/CacheFile.hpp @@ -0,0 +1,83 @@ +// Copyright (C) 2019 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include "Error.hpp" + +#include +#include +#include +#include +#include +#include +#include + +class CacheFile +{ +public: + explicit CacheFile(const std::string& path); + + CacheFile(const CacheFile&) = delete; + CacheFile& operator=(const CacheFile&) = delete; + + const std::string& path() const; + const struct stat& stat() const; + +private: + const std::string m_path; + mutable struct stat m_stat; + mutable bool m_stated = false; +}; + +inline CacheFile::CacheFile(const std::string& path) : m_path(path) +{ +} + +inline const std::string& +CacheFile::path() const +{ + return m_path; +} + +inline const struct stat& +CacheFile::stat() const +{ + if (!m_stated) { +#ifdef _WIN32 + int result = ::stat(m_path.c_str(), &m_stat); +#else + int result = lstat(m_path.c_str(), &m_stat); +#endif + if (result != 0) { + if (errno != ENOENT && errno != ESTALE) { + throw Error( + fmt::format("lstat {} failed: {}", m_path, strerror(errno))); + } + + // The file is missing, so just zero fill the stat structure. This will + // make e.g. S_ISREG(stat().st_mode) return false and stat().st_mtime + // will be, etc. + memset(&m_stat, '\0', sizeof(m_stat)); + } + + m_stated = true; + } + + return m_stat; +} diff --git a/src/util.cpp b/src/util.cpp index 0c46fea4e..f93ba3c1e 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -1706,6 +1706,52 @@ time_seconds(void) #endif } +namespace { + +void +get_cache_files_internal(const std::string& dir, + uint8_t level, + const util::ProgressReceiver& progress_receiver, + std::vector>& files) +{ + DIR* d = opendir(dir.c_str()); + if (!d) { + return; + } + + std::vector directories; + dirent* de; + while ((de = readdir(d))) { + std::string name = de->d_name; + if (name == "" || name == "." || name == ".." || name == "CACHEDIR.TAG" + || name == "stats" || util::starts_with(name, ".nfs")) { + continue; + } + + if (name.length() == 1) { + directories.push_back(name); + } else { + files.push_back( + std::make_shared(fmt::format("{}/{}", dir, name))); + } + } + closedir(d); + + if (level == 1) { + progress_receiver(1.0 / (directories.size() + 1)); + } + + for (size_t i = 0; i < directories.size(); ++i) { + get_cache_files_internal( + dir + "/" + directories[i], level + 1, progress_receiver, files); + if (level == 1) { + progress_receiver(1.0 * (i + 1) / (directories.size() + 1)); + } + } +} + +} // namespace + namespace util { std::string @@ -1773,6 +1819,14 @@ ends_with(const std::string& string, const std::string& suffix) == 0; } +void +get_level_1_files(const std::string& dir, + const ProgressReceiver& progress_receiver, + std::vector>& files) +{ + get_cache_files_internal(dir, 1, progress_receiver, files); +} + std::string read_file(const std::string& path) { diff --git a/src/util.hpp b/src/util.hpp index dc3bf86fc..662db14d9 100644 --- a/src/util.hpp +++ b/src/util.hpp @@ -20,10 +20,17 @@ #include "system.hpp" +#include "CacheFile.hpp" + +#include +#include #include +#include namespace util { +typedef std::function ProgressReceiver; +typedef std::function)> CacheFileVisitor; // Get base name of path. std::string base_name(const std::string& path); @@ -38,6 +45,25 @@ std::string dir_name(const std::string& path); // Return true if suffix is a suffix of string. bool ends_with(const std::string& string, const std::string& suffix); +// Get a list of files in a level 1 subdirectory of the cache. +// +// The function works under the assumption that directory entries with one +// character names (except ".") are subdirectories and that there are no other +// subdirectories. +// +// Files ignored: +// - CACHEDIR.TAG +// - stats +// - .nfs* (temporary NFS files that may be left for open but deleted files). +// +// Parameters: +// - dir: The directory to traverse recursively. +// - progress_receiver: Function that will be called for progress updates. +// - files: Found files. +void get_level_1_files(const std::string& dir, + const ProgressReceiver& progress_receiver, + std::vector>& files); + // Read file data as a string. // // Throws Error on error. diff --git a/unittest/test_util.cpp b/unittest/test_util.cpp index 7fabf181a..bd4640f4d 100644 --- a/unittest/test_util.cpp +++ b/unittest/test_util.cpp @@ -71,6 +71,57 @@ TEST_CASE("util::ends_with") CHECK_FALSE(util::ends_with("x", "xy")); } +TEST_CASE("util::get_level_1_files") +{ + util::create_dir("e/m/p/t/y"); + + util::create_dir("0/1"); + util::create_dir("0/f/c"); + util::write_file("0/file_a", ""); + util::write_file("0/1/file_b", "1"); + util::write_file("0/1/file_c", "12"); + util::write_file("0/f/c/file_d", "123"); + + std::vector> files; + auto null_receiver = [](double) {}; + + SECTION("nonexistent subdirectory") + { + util::get_level_1_files("2", null_receiver, files); + CHECK(files.empty()); + } + + SECTION("empty subdirectory") + { + util::get_level_1_files("e", null_receiver, files); + CHECK(files.empty()); + } + + SECTION("simple case") + { + util::get_level_1_files("0", null_receiver, files); + REQUIRE(files.size() == 4); + + // Files within a level are in arbitrary order, sort them to be able to + // verify them. + std::sort(files.begin(), + files.end(), + [](const std::shared_ptr& f1, + const std::shared_ptr& f2) { + return f1->path() < f2->path(); + }); + + CHECK(files[0]->path() == "0/1/file_b"); + CHECK(files[0]->stat().st_size == 1); + CHECK(files[1]->path() == "0/1/file_c"); + CHECK(files[1]->stat().st_size == 2); + CHECK(files[2]->path() == "0/f/c/file_d"); + CHECK(files[2]->stat().st_size == 3); + CHECK(files[3]->path() == "0/file_a"); + CHECK(files[3]->stat().st_size == 0); + } +} + TEST_CASE("util::read_file and util::write_file") { util::write_file("test", "foo\nbar\n");