AtomicFile.cpp
CacheEntryReader.cpp
CacheEntryWriter.cpp
- CacheFile.cpp
Config.cpp
Context.cpp
Counters.cpp
argprocessing.cpp
assertions.cpp
ccache.cpp
- cleanup.cpp
compopt.cpp
- compress.cpp
execute.cpp
hashutil.cpp
language.cpp
#endif
}
-void
-for_each_level_1_subdir(const std::string& cache_dir,
- const SubdirVisitor& visitor,
- const ProgressReceiver& progress_receiver)
-{
- for (int i = 0; i <= 0xF; i++) {
- double progress = 1.0 * i / 16;
- progress_receiver(progress);
- std::string subdir_path = FMT("{}/{:x}", cache_dir, i);
- visitor(subdir_path, [&](double inner_progress) {
- progress_receiver(progress + inner_progress / 16);
- });
- }
- progress_receiver(1.0);
-}
-
std::string
format_argv_for_logging(const char* const* argv)
{
}
}
-std::vector<CacheFile>
-get_level_1_files(const std::string& dir,
- const ProgressReceiver& progress_receiver)
-{
- std::vector<CacheFile> files;
-
- if (!Stat::stat(dir)) {
- return files;
- }
-
- size_t level_2_directories = 0;
-
- Util::traverse(dir, [&](const std::string& path, bool is_dir) {
- auto name = Util::base_name(path);
- if (name == "CACHEDIR.TAG" || name == "stats" || name.starts_with(".nfs")) {
- return;
- }
-
- if (!is_dir) {
- files.emplace_back(path);
- } else if (path != dir
- && path.find('/', dir.size() + 1) == std::string::npos) {
- ++level_2_directories;
- progress_receiver(level_2_directories / 16.0);
- }
- });
-
- progress_receiver(1.0);
- return files;
-}
-
std::string
get_home_directory()
{
#pragma once
-#include "CacheFile.hpp"
-
+#include <Stat.hpp>
#include <util/Tokenizer.hpp>
#include "third_party/nonstd/optional.hpp"
namespace Util {
using DataReceiver = std::function<void(const void* data, size_t size)>;
-using ProgressReceiver = std::function<void(double progress)>;
-using SubdirVisitor = std::function<void(
- const std::string& dir_path, const ProgressReceiver& progress_receiver)>;
using TraverseVisitor =
std::function<void(const std::string& path, bool is_dir)>;
// Returns 0 on success, an error number otherwise.
int fallocate(int fd, long new_size);
-// Call a function for each subdir (0-9a-f) in the cache.
-//
-// Parameters:
-// - cache_dir: Path to the cache directory.
-// - visitor: Function to call with directory path and progress_receiver as
-// arguments.
-// - progress_receiver: Function that will be called for progress updates.
-void for_each_level_1_subdir(const std::string& cache_dir,
- const SubdirVisitor& visitor,
- const ProgressReceiver& progress_receiver);
-
// Format `argv` as a simple string for logging purposes. That is, the result is
// not intended to be machine parsable. `argv` must be terminated by a nullptr.
std::string format_argv_for_logging(const char* const* argv);
// `path` has no file extension, an empty string_view is returned.
nonstd::string_view get_extension(nonstd::string_view path);
-// Get a list of files in a level 1 subdirectory of the cache.
-//
-// The function works under the assumption that directory entries with one
-// character names (except ".") are subdirectories and that there are no other
-// subdirectories.
-//
-// Files ignored:
-// - CACHEDIR.TAG
-// - stats
-// - .nfs* (temporary NFS files that may be left for open but deleted files).
-//
-// Parameters:
-// - dir: The directory to traverse recursively.
-// - progress_receiver: Function that will be called for progress updates.
-std::vector<CacheFile>
-get_level_1_files(const std::string& dir,
- const ProgressReceiver& progress_receiver);
-
// Return the current user's home directory, or throw `Fatal` if it can't
// be determined.
std::string get_home_directory();
#include "Util.hpp"
#include "Win32Util.hpp"
#include "argprocessing.hpp"
-#include "cleanup.hpp"
#include "compopt.hpp"
-#include "compress.hpp"
#include "execute.hpp"
#include "fmtmacros.hpp"
#include "hashutil.hpp"
case EVICT_OLDER_THAN: {
auto seconds = Util::parse_duration(arg);
ProgressBar progress_bar("Evicting...");
- clean_old(
- ctx, [&](double progress) { progress_bar.update(progress); }, seconds);
+ ctx.storage.primary().clean_old(
+ [&](double progress) { progress_bar.update(progress); }, seconds);
if (isatty(STDOUT_FILENO)) {
PRINT_RAW(stdout, "\n");
}
case 'c': // --cleanup
{
ProgressBar progress_bar("Cleaning...");
- clean_up_all(ctx.config,
- [&](double progress) { progress_bar.update(progress); });
+ ctx.storage.primary().clean_all(
+ [&](double progress) { progress_bar.update(progress); });
if (isatty(STDOUT_FILENO)) {
PRINT_RAW(stdout, "\n");
}
case 'C': // --clear
{
ProgressBar progress_bar("Clearing...");
- wipe_all(ctx, [&](double progress) { progress_bar.update(progress); });
+ ctx.storage.primary().wipe_all(
+ [&](double progress) { progress_bar.update(progress); });
if (isatty(STDOUT_FILENO)) {
PRINT_RAW(stdout, "\n");
}
+#ifdef INODE_CACHE_SUPPORTED
+ ctx.inode_cache.drop();
+#endif
break;
}
case 'x': // --show-compression
{
ProgressBar progress_bar("Scanning...");
- compress_stats(ctx.config,
- [&](double progress) { progress_bar.update(progress); });
+ ctx.storage.primary().print_compression_statistics(
+ [&](double progress) { progress_bar.update(progress); });
break;
}
}
ProgressBar progress_bar("Recompressing...");
- compress_recompress(ctx, wanted_level, [&](double progress) {
- progress_bar.update(progress);
- });
+ ctx.storage.primary().recompress(
+ wanted_level, [&](double progress) { progress_bar.update(progress); });
break;
}
+++ /dev/null
-// Copyright (C) 2019-2021 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#pragma once
-
-#include "Util.hpp"
-
-#include <cstdint>
-#include <string>
-
-class Config;
-class Context;
-
-void clean_old(const Context& ctx,
- const Util::ProgressReceiver& progress_receiver,
- uint64_t max_age);
-
-void clean_up_dir(const std::string& subdir,
- uint64_t max_size,
- uint64_t max_files,
- uint64_t max_age,
- const Util::ProgressReceiver& progress_receiver);
-
-void clean_up_all(const Config& config,
- const Util::ProgressReceiver& progress_receiver);
-
-void wipe_all(const Context& ctx,
- const Util::ProgressReceiver& progress_receiver);
+++ /dev/null
-// Copyright (C) 2019-2021 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#pragma once
-
-#include "Util.hpp"
-
-#include "third_party/nonstd/optional.hpp"
-
-class Config;
-class Context;
-
-void compress_stats(const Config& config,
- const Util::ProgressReceiver& progress_receiver);
-
-// Recompress the cache.
-//
-// Arguments:
-// - ctx: The context.
-// - level: Target compression level (positive or negative value for actual
-// level, 0 for default level and nonstd::nullopt for no compression).
-// - progress_receiver: Function that will be called for progress updates.
-void compress_recompress(Context& ctx,
- nonstd::optional<int8_t> level,
- const Util::ProgressReceiver& progress_receiver);
#pragma once
-#include "types.hpp"
-
#include <core/types.hpp>
#include <storage/primary/PrimaryStorage.hpp>
+#include <storage/types.hpp>
#include <third_party/nonstd/optional.hpp>
set(
sources
+ ${CMAKE_CURRENT_SOURCE_DIR}/CacheFile.cpp
${CMAKE_CURRENT_SOURCE_DIR}/PrimaryStorage.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/PrimaryStorage_cleanup.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/PrimaryStorage_compress.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/util.cpp
)
target_sources(ccache_lib PRIVATE ${sources})
#include "CacheFile.hpp"
-#include "Manifest.hpp"
-#include "Result.hpp"
-
+#include <Manifest.hpp>
+#include <Result.hpp>
#include <util/string.hpp>
const Stat&
#pragma once
-#include "Stat.hpp"
+#include <Stat.hpp>
-#include "third_party/nonstd/optional.hpp"
+#include <third_party/nonstd/optional.hpp>
#include <string>
#include <Statistics.hpp>
#include <Util.hpp>
#include <assertions.hpp>
-#include <cleanup.hpp>
#include <core/exceptions.hpp>
#include <core/wincompat.hpp>
#include <fmtmacros.hpp>
void
PrimaryStorage::initialize()
{
- MTR_BEGIN("primary_storage", "clean_up_internal_tempdir");
+ MTR_BEGIN("primary_storage", "clean_internal_tempdir");
if (m_config.temporary_dir() == m_config.cache_dir() + "/tmp") {
- clean_up_internal_tempdir();
+ clean_internal_tempdir();
}
- MTR_END("primary_storage", "clean_up_internal_tempdir");
+ MTR_END("primary_storage", "clean_internal_tempdir");
}
void
const uint64_t max_size = round(m_config.max_size() * factor);
const uint32_t max_files = round(m_config.max_files() * factor);
const time_t max_age = 0;
- clean_up_dir(
- subdir, max_size, max_files, max_age, [](double /*progress*/) {});
+ clean_dir(subdir, max_size, max_files, max_age, [](double /*progress*/) {});
}
}
}
void
-PrimaryStorage::clean_up_internal_tempdir()
+PrimaryStorage::clean_internal_tempdir()
{
const time_t now = time(nullptr);
const auto dir_st = Stat::stat(m_config.cache_dir(), Stat::OnError::log);
#pragma once
+#include "util.hpp"
+
#include <Counters.hpp>
#include <Digest.hpp>
#include <core/types.hpp>
void initialize();
void finalize();
+ // --- Cache entry handling ---
+
// Returns a path to a file containing the value.
nonstd::optional<std::string> get(const Digest& key,
core::CacheEntryType type) const;
void remove(const Digest& key, core::CacheEntryType type);
+ // --- Statistics ---
+
void increment_statistic(Statistic statistic, int64_t value = 1);
// Return a machine-readable string representing the final ccache result, or
// nullopt if there was no result.
nonstd::optional<std::string> get_result_message() const;
+ // --- Cleanup ---
+
+ void clean_old(const ProgressReceiver& progress_receiver, uint64_t max_age);
+
+ void clean_dir(const std::string& subdir,
+ uint64_t max_size,
+ uint64_t max_files,
+ uint64_t max_age,
+ const ProgressReceiver& progress_receiver);
+
+ void clean_all(const ProgressReceiver& progress_receiver);
+
+ void wipe_all(const ProgressReceiver& progress_receiver);
+
+ // --- Compression ---
+
+ void print_compression_statistics(const ProgressReceiver& progress_receiver);
+
+ void recompress(nonstd::optional<int8_t> level,
+ const ProgressReceiver& progress_receiver);
+
private:
const Config& m_config;
LookUpCacheFileResult look_up_cache_file(const Digest& key,
core::CacheEntryType type) const;
- void clean_up_internal_tempdir();
+ void clean_internal_tempdir();
nonstd::optional<Counters>
update_stats_and_maybe_move_cache_file(const Digest& key,
// this program; if not, write to the Free Software Foundation, Inc., 51
// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-#include "cleanup.hpp"
-
-#include "CacheFile.hpp"
-#include "Config.hpp"
-#include "Context.hpp"
-#include "Logging.hpp"
-#include "Statistics.hpp"
-#include "Util.hpp"
-
+#include "PrimaryStorage.hpp"
+
+#include <Config.hpp>
+#include <Context.hpp>
+#include <Logging.hpp>
+#include <Statistics.hpp>
+#include <Util.hpp>
+#include <storage/primary/CacheFile.hpp>
+#include <storage/primary/util.hpp>
#include <util/string.hpp>
#ifdef INODE_CACHE_SUPPORTED
-# include "InodeCache.hpp"
+# include <InodeCache.hpp>
#endif
#include <algorithm>
+namespace storage {
+namespace primary {
+
static void
delete_file(const std::string& path,
- uint64_t size,
+ const uint64_t size,
uint64_t* cache_size,
uint64_t* files_in_cache)
{
- bool deleted = Util::unlink_safe(path, Util::UnlinkLog::ignore_failure);
+ const bool deleted = Util::unlink_safe(path, Util::UnlinkLog::ignore_failure);
if (!deleted && errno != ENOENT && errno != ESTALE) {
LOG("Failed to unlink {} ({})", path, strerror(errno));
} else if (cache_size && files_in_cache) {
static void
update_counters(const std::string& dir,
- uint64_t files_in_cache,
- uint64_t cache_size,
- bool cleanup_performed)
+ const uint64_t files_in_cache,
+ const uint64_t cache_size,
+ const bool cleanup_performed)
{
const std::string stats_file = dir + "/stats";
Statistics::update(stats_file, [=](auto& cs) {
}
void
-clean_old(const Context& ctx,
- const Util::ProgressReceiver& progress_receiver,
- uint64_t max_age)
+PrimaryStorage::clean_old(const ProgressReceiver& progress_receiver,
+ const uint64_t max_age)
{
- Util::for_each_level_1_subdir(
- ctx.config.cache_dir(),
- [&](const auto& subdir, const auto& sub_progress_receiver) {
- clean_up_dir(subdir, 0, 0, max_age, sub_progress_receiver);
+ for_each_level_1_subdir(
+ m_config.cache_dir(),
+ [&](const std::string& subdir,
+ const ProgressReceiver& sub_progress_receiver) {
+ clean_dir(subdir, 0, 0, max_age, sub_progress_receiver);
},
progress_receiver);
}
// Clean up one cache subdirectory.
void
-clean_up_dir(const std::string& subdir,
- uint64_t max_size,
- uint64_t max_files,
- uint64_t max_age,
- const Util::ProgressReceiver& progress_receiver)
+PrimaryStorage::clean_dir(const std::string& subdir,
+ const uint64_t max_size,
+ const uint64_t max_files,
+ const uint64_t max_age,
+ const ProgressReceiver& progress_receiver)
{
LOG("Cleaning up cache directory {}", subdir);
- std::vector<CacheFile> files = Util::get_level_1_files(
+ std::vector<CacheFile> files = get_level_1_files(
subdir, [&](double progress) { progress_receiver(progress / 3); });
uint64_t cache_size = 0;
// Clean up all cache subdirectories.
void
-clean_up_all(const Config& config,
- const Util::ProgressReceiver& progress_receiver)
+PrimaryStorage::clean_all(const ProgressReceiver& progress_receiver)
{
- Util::for_each_level_1_subdir(
- config.cache_dir(),
- [&](const auto& subdir, const auto& sub_progress_receiver) {
- clean_up_dir(subdir,
- config.max_size() / 16,
- config.max_files() / 16,
- 0,
- sub_progress_receiver);
+ for_each_level_1_subdir(
+ m_config.cache_dir(),
+ [&](const std::string& subdir,
+ const ProgressReceiver& sub_progress_receiver) {
+ clean_dir(subdir,
+ m_config.max_size() / 16,
+ m_config.max_files() / 16,
+ 0,
+ sub_progress_receiver);
},
progress_receiver);
}
// Wipe one cache subdirectory.
static void
-wipe_dir(const std::string& subdir,
- const Util::ProgressReceiver& progress_receiver)
+wipe_dir(const std::string& subdir, const ProgressReceiver& progress_receiver)
{
LOG("Clearing out cache directory {}", subdir);
- const std::vector<CacheFile> files = Util::get_level_1_files(
+ const std::vector<CacheFile> files = get_level_1_files(
subdir, [&](double progress) { progress_receiver(progress / 2); });
for (size_t i = 0; i < files.size(); ++i) {
// Wipe all cached files in all subdirectories.
void
-wipe_all(const Context& ctx, const Util::ProgressReceiver& progress_receiver)
+PrimaryStorage::wipe_all(const ProgressReceiver& progress_receiver)
{
- Util::for_each_level_1_subdir(
- ctx.config.cache_dir(), wipe_dir, progress_receiver);
-#ifdef INODE_CACHE_SUPPORTED
- ctx.inode_cache.drop();
-#endif
+ for_each_level_1_subdir(m_config.cache_dir(), wipe_dir, progress_receiver);
}
+
+} // namespace primary
+} // namespace storage
// this program; if not, write to the Free Software Foundation, Inc., 51
// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-#include "compress.hpp"
-
-#include "AtomicFile.hpp"
-#include "CacheEntryReader.hpp"
-#include "CacheEntryWriter.hpp"
-#include "Context.hpp"
-#include "File.hpp"
-#include "Logging.hpp"
-#include "Manifest.hpp"
-#include "Result.hpp"
-#include "Statistics.hpp"
-#include "ThreadPool.hpp"
-#include "assertions.hpp"
-#include "fmtmacros.hpp"
-
+#include "PrimaryStorage.hpp"
+
+#include <AtomicFile.hpp>
+#include <CacheEntryReader.hpp>
+#include <CacheEntryWriter.hpp>
+#include <Context.hpp>
+#include <File.hpp>
+#include <Logging.hpp>
+#include <Manifest.hpp>
+#include <Result.hpp>
+#include <Statistics.hpp>
+#include <ThreadPool.hpp>
+#include <assertions.hpp>
#include <compression/ZstdCompressor.hpp>
#include <core/exceptions.hpp>
#include <core/wincompat.hpp>
+#include <fmtmacros.hpp>
#include <util/string.hpp>
-#include "third_party/fmt/core.h"
+#include <third_party/fmt/core.h>
#ifdef HAVE_UNISTD_H
# include <unistd.h>
#include <string>
#include <thread>
-using nonstd::optional;
+namespace storage {
+namespace primary {
namespace {
};
void
-RecompressionStatistics::update(uint64_t content_size,
- uint64_t old_size,
- uint64_t new_size,
- uint64_t incompressible_size)
+RecompressionStatistics::update(const uint64_t content_size,
+ const uint64_t old_size,
+ const uint64_t new_size,
+ const uint64_t incompressible_size)
{
std::unique_lock<std::mutex> lock(m_mutex);
m_incompressible_size += incompressible_size;
return m_incompressible_size;
}
-File
-open_file(const std::string& path, const char* mode)
+} // namespace
+
+static File
+open_file(const std::string& path, const char* const mode)
{
File f(path, mode);
if (!f) {
return f;
}
-std::unique_ptr<CacheEntryReader>
-create_reader(const CacheFile& cache_file, FILE* stream)
+static std::unique_ptr<CacheEntryReader>
+create_reader(const CacheFile& cache_file, FILE* const stream)
{
if (cache_file.type() == CacheFile::Type::unknown) {
throw core::Error("unknown file type for {}", cache_file.path());
ASSERT(false);
}
-std::unique_ptr<CacheEntryWriter>
-create_writer(FILE* stream,
+static std::unique_ptr<CacheEntryWriter>
+create_writer(FILE* const stream,
const CacheEntryReader& reader,
- compression::Type compression_type,
- int8_t compression_level)
+ const compression::Type compression_type,
+ const int8_t compression_level)
{
return std::make_unique<CacheEntryWriter>(stream,
reader.magic(),
reader.payload_size());
}
-void
+static void
recompress_file(RecompressionStatistics& statistics,
const std::string& stats_file,
const CacheFile& cache_file,
- optional<int8_t> level)
+ const nonstd::optional<int8_t> level)
{
auto file = open_file(cache_file.path(), "rb");
auto reader = create_reader(cache_file, file.get());
- auto old_stat = Stat::stat(cache_file.path(), Stat::OnError::log);
- uint64_t content_size = reader->content_size();
- int8_t wanted_level =
+ const auto old_stat = Stat::stat(cache_file.path(), Stat::OnError::log);
+ const uint64_t content_size = reader->content_size();
+ const int8_t wanted_level =
level
? (*level == 0 ? compression::ZstdCompressor::default_compression_level
: *level)
file.close();
atomic_new_file.commit();
- auto new_stat = Stat::stat(cache_file.path(), Stat::OnError::log);
+ const auto new_stat = Stat::stat(cache_file.path(), Stat::OnError::log);
Statistics::update(stats_file, [=](auto& cs) {
cs.increment(Statistic::cache_size_kibibyte,
LOG("Recompression of {} done", cache_file.path());
}
-} // namespace
-
void
-compress_stats(const Config& config,
- const Util::ProgressReceiver& progress_receiver)
+PrimaryStorage::print_compression_statistics(
+ const ProgressReceiver& progress_receiver)
{
uint64_t on_disk_size = 0;
uint64_t compr_size = 0;
uint64_t content_size = 0;
uint64_t incompr_size = 0;
- Util::for_each_level_1_subdir(
- config.cache_dir(),
+ for_each_level_1_subdir(
+ m_config.cache_dir(),
[&](const auto& subdir, const auto& sub_progress_receiver) {
- const std::vector<CacheFile> files = Util::get_level_1_files(
+ const std::vector<CacheFile> files = get_level_1_files(
subdir, [&](double progress) { sub_progress_receiver(progress / 2); });
for (size_t i = 0; i < files.size(); ++i) {
PRINT_RAW(stdout, "\n\n");
}
- double ratio =
+ const double ratio =
compr_size > 0 ? static_cast<double>(content_size) / compr_size : 0.0;
- double savings = ratio > 0.0 ? 100.0 - (100.0 / ratio) : 0.0;
+ const double savings = ratio > 0.0 ? 100.0 - (100.0 / ratio) : 0.0;
- std::string on_disk_size_str = Util::format_human_readable_size(on_disk_size);
- std::string cache_size_str =
+ const std::string on_disk_size_str =
+ Util::format_human_readable_size(on_disk_size);
+ const std::string cache_size_str =
Util::format_human_readable_size(compr_size + incompr_size);
- std::string compr_size_str = Util::format_human_readable_size(compr_size);
- std::string content_size_str = Util::format_human_readable_size(content_size);
- std::string incompr_size_str = Util::format_human_readable_size(incompr_size);
+ const std::string compr_size_str =
+ Util::format_human_readable_size(compr_size);
+ const std::string content_size_str =
+ Util::format_human_readable_size(content_size);
+ const std::string incompr_size_str =
+ Util::format_human_readable_size(incompr_size);
PRINT(stdout,
"Total data: {:>8s} ({} disk blocks)\n",
}
void
-compress_recompress(Context& ctx,
- optional<int8_t> level,
- const Util::ProgressReceiver& progress_receiver)
+PrimaryStorage::recompress(const nonstd::optional<int8_t> level,
+ const ProgressReceiver& progress_receiver)
{
const size_t threads = std::thread::hardware_concurrency();
const size_t read_ahead = 2 * threads;
ThreadPool thread_pool(threads, read_ahead);
RecompressionStatistics statistics;
- Util::for_each_level_1_subdir(
- ctx.config.cache_dir(),
+ for_each_level_1_subdir(
+ m_config.cache_dir(),
[&](const auto& subdir, const auto& sub_progress_receiver) {
std::vector<CacheFile> files =
- Util::get_level_1_files(subdir, [&](double progress) {
+ get_level_1_files(subdir, [&](double progress) {
sub_progress_receiver(0.1 * progress);
});
}
if (util::ends_with(subdir, "f")) {
- // Wait here instead of after Util::for_each_level_1_subdir to avoid
+ // Wait here instead of after for_each_level_1_subdir to avoid
// updating the progress bar to 100% before all work is done.
thread_pool.shut_down();
}
PRINT_RAW(stdout, "\n\n");
}
- double old_ratio =
+ const double old_ratio =
statistics.old_size() > 0
? static_cast<double>(statistics.content_size()) / statistics.old_size()
: 0.0;
- double old_savings = old_ratio > 0.0 ? 100.0 - (100.0 / old_ratio) : 0.0;
- double new_ratio =
+ const double old_savings =
+ old_ratio > 0.0 ? 100.0 - (100.0 / old_ratio) : 0.0;
+ const double new_ratio =
statistics.new_size() > 0
? static_cast<double>(statistics.content_size()) / statistics.new_size()
: 0.0;
- double new_savings = new_ratio > 0.0 ? 100.0 - (100.0 / new_ratio) : 0.0;
- int64_t size_difference = static_cast<int64_t>(statistics.new_size())
- - static_cast<int64_t>(statistics.old_size());
+ const double new_savings =
+ new_ratio > 0.0 ? 100.0 - (100.0 / new_ratio) : 0.0;
+ const int64_t size_difference = static_cast<int64_t>(statistics.new_size())
+ - static_cast<int64_t>(statistics.old_size());
- std::string old_compr_size_str =
+ const std::string old_compr_size_str =
Util::format_human_readable_size(statistics.old_size());
- std::string new_compr_size_str =
+ const std::string new_compr_size_str =
Util::format_human_readable_size(statistics.new_size());
- std::string content_size_str =
+ const std::string content_size_str =
Util::format_human_readable_size(statistics.content_size());
- std::string incompr_size_str =
+ const std::string incompr_size_str =
Util::format_human_readable_size(statistics.incompressible_size());
- std::string size_difference_str =
+ const std::string size_difference_str =
FMT("{}{}",
size_difference < 0 ? "-" : (size_difference > 0 ? "+" : " "),
Util::format_human_readable_size(
new_savings);
PRINT(stdout, "Size change: {:>9s}\n", size_difference_str);
}
+
+} // namespace primary
+} // namespace storage
--- /dev/null
+// Copyright (C) 2021 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "util.hpp"
+
+#include <Util.hpp>
+#include <fmtmacros.hpp>
+
+namespace storage {
+namespace primary {
+
+void
+for_each_level_1_subdir(const std::string& cache_dir,
+ const SubdirVisitor& visitor,
+ const ProgressReceiver& progress_receiver)
+{
+ for (int i = 0; i <= 0xF; i++) {
+ double progress = 1.0 * i / 16;
+ progress_receiver(progress);
+ std::string subdir_path = FMT("{}/{:x}", cache_dir, i);
+ visitor(subdir_path, [&](double inner_progress) {
+ progress_receiver(progress + inner_progress / 16);
+ });
+ }
+ progress_receiver(1.0);
+}
+
+std::vector<CacheFile>
+get_level_1_files(const std::string& dir,
+ const ProgressReceiver& progress_receiver)
+{
+ std::vector<CacheFile> files;
+
+ if (!Stat::stat(dir)) {
+ return files;
+ }
+
+ size_t level_2_directories = 0;
+
+ Util::traverse(dir, [&](const std::string& path, bool is_dir) {
+ auto name = Util::base_name(path);
+ if (name == "CACHEDIR.TAG" || name == "stats" || name.starts_with(".nfs")) {
+ return;
+ }
+
+ if (!is_dir) {
+ files.emplace_back(path);
+ } else if (path != dir
+ && path.find('/', dir.size() + 1) == std::string::npos) {
+ ++level_2_directories;
+ progress_receiver(level_2_directories / 16.0);
+ }
+ });
+
+ progress_receiver(1.0);
+ return files;
+}
+
+} // namespace primary
+} // namespace storage
--- /dev/null
+// Copyright (C) 2021 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#pragma once
+
+#include <storage/primary/CacheFile.hpp>
+
+#include <functional>
+#include <string>
+#include <vector>
+
+namespace storage {
+namespace primary {
+
+using ProgressReceiver = std::function<void(double progress)>;
+using SubdirVisitor = std::function<void(
+ const std::string& dir_path, const ProgressReceiver& progress_receiver)>;
+
+// Call a function for each subdir (0-9a-f) in the cache.
+//
+// Parameters:
+// - cache_dir: Path to the cache directory.
+// - visitor: Function to call with directory path and progress_receiver as
+// arguments.
+// - progress_receiver: Function that will be called for progress updates.
+void for_each_level_1_subdir(const std::string& cache_dir,
+ const SubdirVisitor& visitor,
+ const ProgressReceiver& progress_receiver);
+
+// Get a list of files in a level 1 subdirectory of the cache.
+//
+// The function works under the assumption that directory entries with one
+// character names (except ".") are subdirectories and that there are no other
+// subdirectories.
+//
+// Files ignored:
+// - CACHEDIR.TAG
+// - stats
+// - .nfs* (temporary NFS files that may be left for open but deleted files).
+//
+// Parameters:
+// - dir: The directory to traverse recursively.
+// - progress_receiver: Function that will be called for progress updates.
+std::vector<CacheFile>
+get_level_1_files(const std::string& dir,
+ const ProgressReceiver& progress_receiver);
+
+} // namespace primary
+} // namespace storage
#pragma once
-#include "SecondaryStorage.hpp"
+#include <storage/secondary/SecondaryStorage.hpp>
namespace storage {
namespace secondary {
#pragma once
-#include "SecondaryStorage.hpp"
+#include <storage/secondary/SecondaryStorage.hpp>
namespace storage {
namespace secondary {
#pragma once
-#include "SecondaryStorage.hpp"
+#include <storage/secondary/SecondaryStorage.hpp>
namespace storage {
namespace secondary {
test_compopt.cpp
test_compression_types.cpp
test_hashutil.cpp
+ test_storage_primary_util.cpp
test_util_Tokenizer.cpp
test_util_expected.cpp
test_util_path.cpp
CHECK(Stat::stat(filename).size() == 20000);
}
-TEST_CASE("Util::for_each_level_1_subdir")
-{
- std::vector<std::string> actual;
- Util::for_each_level_1_subdir(
- "cache_dir",
- [&](const auto& subdir, const auto&) { actual.push_back(subdir); },
- [](double) {});
-
- std::vector<std::string> expected = {
- "cache_dir/0",
- "cache_dir/1",
- "cache_dir/2",
- "cache_dir/3",
- "cache_dir/4",
- "cache_dir/5",
- "cache_dir/6",
- "cache_dir/7",
- "cache_dir/8",
- "cache_dir/9",
- "cache_dir/a",
- "cache_dir/b",
- "cache_dir/c",
- "cache_dir/d",
- "cache_dir/e",
- "cache_dir/f",
- };
- CHECK(actual == expected);
-}
-
TEST_CASE("Util::format_argv_for_logging")
{
const char* argv_0[] = {nullptr};
return path;
}
-TEST_CASE("Util::get_level_1_files")
-{
- TestContext test_context;
-
- Util::create_dir("e/m/p/t/y");
-
- Util::create_dir("0/1");
- Util::create_dir("0/f/c");
- Util::write_file("0/file_a", "");
- Util::write_file("0/1/file_b", "1");
- Util::write_file("0/1/file_c", "12");
- Util::write_file("0/f/c/file_d", "123");
-
- auto null_receiver = [](double) {};
-
- SUBCASE("nonexistent subdirectory")
- {
- const auto files = Util::get_level_1_files("2", null_receiver);
- CHECK(files.empty());
- }
-
- SUBCASE("empty subdirectory")
- {
- const auto files = Util::get_level_1_files("e", null_receiver);
- CHECK(files.empty());
- }
-
- SUBCASE("simple case")
- {
- auto files = Util::get_level_1_files("0", null_receiver);
- REQUIRE(files.size() == 4);
-
- // Files within a level are in arbitrary order, sort them to be able to
- // verify them.
- std::sort(files.begin(), files.end(), [](const auto& f1, const auto& f2) {
- return f1.path() < f2.path();
- });
-
- CHECK(files[0].path() == os_path("0/1/file_b"));
- CHECK(files[0].lstat().size() == 1);
- CHECK(files[1].path() == os_path("0/1/file_c"));
- CHECK(files[1].lstat().size() == 2);
- CHECK(files[2].path() == os_path("0/f/c/file_d"));
- CHECK(files[2].lstat().size() == 3);
- CHECK(files[3].path() == os_path("0/file_a"));
- CHECK(files[3].lstat().size() == 0);
- }
-}
-
TEST_CASE("Util::get_relative_path")
{
#ifdef _WIN32
--- /dev/null
+// Copyright (C) 2021 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "TestUtil.hpp"
+
+#include <Util.hpp>
+#include <storage/primary/util.hpp>
+
+#include <third_party/doctest.h>
+
+#include <string>
+
+using TestUtil::TestContext;
+
+static inline std::string
+os_path(std::string path)
+{
+#if defined(_WIN32) && !defined(HAVE_DIRENT_H)
+ std::replace(path.begin(), path.end(), '/', '\\');
+#endif
+
+ return path;
+}
+
+TEST_SUITE_BEGIN("storage::primary::util");
+
+TEST_CASE("storage::primary::for_each_level_1_subdir")
+{
+ std::vector<std::string> actual;
+ storage::primary::for_each_level_1_subdir(
+ "cache_dir",
+ [&](const auto& subdir, const auto&) { actual.push_back(subdir); },
+ [](double) {});
+
+ std::vector<std::string> expected = {
+ "cache_dir/0",
+ "cache_dir/1",
+ "cache_dir/2",
+ "cache_dir/3",
+ "cache_dir/4",
+ "cache_dir/5",
+ "cache_dir/6",
+ "cache_dir/7",
+ "cache_dir/8",
+ "cache_dir/9",
+ "cache_dir/a",
+ "cache_dir/b",
+ "cache_dir/c",
+ "cache_dir/d",
+ "cache_dir/e",
+ "cache_dir/f",
+ };
+ CHECK(actual == expected);
+}
+
+TEST_CASE("storage::primary::get_level_1_files")
+{
+ TestContext test_context;
+
+ Util::create_dir("e/m/p/t/y");
+
+ Util::create_dir("0/1");
+ Util::create_dir("0/f/c");
+ Util::write_file("0/file_a", "");
+ Util::write_file("0/1/file_b", "1");
+ Util::write_file("0/1/file_c", "12");
+ Util::write_file("0/f/c/file_d", "123");
+
+ auto null_receiver = [](double) {};
+
+ SUBCASE("nonexistent subdirectory")
+ {
+ const auto files = storage::primary::get_level_1_files("2", null_receiver);
+ CHECK(files.empty());
+ }
+
+ SUBCASE("empty subdirectory")
+ {
+ const auto files = storage::primary::get_level_1_files("e", null_receiver);
+ CHECK(files.empty());
+ }
+
+ SUBCASE("simple case")
+ {
+ auto files = storage::primary::get_level_1_files("0", null_receiver);
+ REQUIRE(files.size() == 4);
+
+ // Files within a level are in arbitrary order, sort them to be able to
+ // verify them.
+ std::sort(files.begin(), files.end(), [](const auto& f1, const auto& f2) {
+ return f1.path() < f2.path();
+ });
+
+ CHECK(files[0].path() == os_path("0/1/file_b"));
+ CHECK(files[0].lstat().size() == 1);
+ CHECK(files[1].path() == os_path("0/1/file_c"));
+ CHECK(files[1].lstat().size() == 2);
+ CHECK(files[2].path() == os_path("0/f/c/file_d"));
+ CHECK(files[2].lstat().size() == 3);
+ CHECK(files[3].path() == os_path("0/file_a"));
+ CHECK(files[3].lstat().size() == 0);
+ }
+}
+
+TEST_SUITE_END();
projects / thirdparty / ccache.git / commitdiff
