});
}
-static std::string
-suffix_from_type(const core::CacheEntryType type)
-{
- switch (type) {
- case core::CacheEntryType::manifest:
- return "M";
-
- case core::CacheEntryType::result:
- return "R";
- }
-
- ASSERT(false);
-}
-
static uint8_t
calculate_wanted_cache_level(const uint64_t files_in_level_1)
{
}
}
+static std::optional<std::string>
+result_path_from_raw_file(const std::string& path)
+{
+ if (path.length() < 3) {
+ return std::nullopt;
+ }
+
+ if (path[path.length() - 3] == '_') {
+ // raw file ends with "_nn" where nn is file number in hex form
+ return path.substr(0, path.length() - 3);
+ }
+
+ if (path[path.length() - 1] == 'W') {
+ // legacy: raw file ends with "nW" where n is file number (0-9)
+ return FMT("{}R", path.substr(0, path.length() - 2));
+ }
+
+ return std::nullopt;
+}
+
static CleanDirResult
clean_dir(
const fs::path& l2_dir,
continue;
}
- if (namespace_ && file_type_from_path(file.path()) == FileType::raw) {
- util::PathString path_str(file.path());
- const std::string result_path =
- FMT("{}R", path_str.str().substr(0, path_str.str().length() - 2));
- raw_files_map[result_path].push_back(file.path());
+ if (namespace_) {
+ auto result_path = result_path_from_raw_file(util::pstr(file.path()));
+ if (result_path) {
+ raw_files_map[*result_path].push_back(file.path());
+ }
}
cache_size += file.size_on_disk();
// For namespace eviction we need to remove raw files based on result
// filename since they don't have a header.
- if (file_type_from_path(file.path()) == FileType::result) {
- const auto entry = raw_files_map.find(util::pstr(file.path()));
- if (entry != raw_files_map.end()) {
- for (const auto& raw_file : entry->second) {
- delete_file(DirEntry(raw_file), cache_size, files_in_cache);
- }
+ const auto entry = raw_files_map.find(util::pstr(file.path()));
+ if (entry != raw_files_map.end()) {
+ for (const auto& raw_file : entry->second) {
+ delete_file(DirEntry(raw_file), cache_size, files_in_cache);
}
}
}
return {counters_before, counters_after};
}
-FileType
-file_type_from_path(const fs::path& path)
-{
- std::string filename = path.filename().string();
- if (util::ends_with(filename, "M")) {
- return FileType::manifest;
- } else if (util::ends_with(filename, "R")) {
- return FileType::result;
- } else if (util::ends_with(filename, "W")) {
- return FileType::raw;
- } else {
- return FileType::unknown;
- }
-}
-
LocalStorage::LocalStorage(const Config& config)
: m_config(config)
{
return;
}
- move_to_wanted_cache_level(*counters, key, type, cache_file.path);
+ move_to_wanted_cache_level(*counters, key, cache_file.path);
// Make sure we have a CACHEDIR.TAG in the cache part of cache_dir. This can
// be done almost anywhere, but we might as well do it near the end as we save
LocalStorage::get_raw_file_path(const fs::path& result_path,
uint8_t file_number)
{
- if (file_number >= 10) {
- // To support more entries in the future, encode to [0-9a-z]. Note that
- // LocalStorage::evict currently assumes that the entry number is
- // represented as one character.
- throw core::Error(FMT("Too high raw file entry number: {}", file_number));
- }
-
- std::string s = result_path.string();
- s.pop_back();
- return FMT("{}{}W", s, file_number);
+ return FMT("{}_{:02x}", result_path, file_number);
}
fs::path
const Hash::Digest& key,
const std::vector<core::result::Serializer::RawFile>& raw_files)
{
+ const size_t k_max_raw_files = 256;
+ if (raw_files.size() > k_max_raw_files) {
+ throw core::Error(
+ FMT("Too many raw files: {} > {}", raw_files.size(), k_max_raw_files));
+ }
+
const auto cache_file = look_up_cache_file(key, core::CacheEntryType::result);
core::ensure_dir_exists(cache_file.path.parent_path());
auto stats_file = get_stats_file(l1_index);
for (const auto& file : files) {
- if (file_type_from_path(file.path()) != FileType::unknown) {
+ if (!util::TemporaryFile::is_tmp_file(file.path())) {
thread_pool.enqueue_detach([&, file, l2_index, stats_file, level] {
try {
DirEntry new_dir_entry = recompressor.recompress(
incompressible_size += file.size_on_disk();
}
});
- } else if (!util::TemporaryFile::is_tmp_file(file.path())) {
- incompressible_size += file.size_on_disk();
}
}
LocalStorage::look_up_cache_file(const Hash::Digest& key,
const core::CacheEntryType type) const
{
- const auto key_string =
- FMT("{}{}", util::format_base16(key), suffix_from_type(type));
+ // Try new format first: base16 without suffix
+ const auto key_string = util::format_base16(key);
for (uint8_t level = k_min_cache_levels; level <= k_max_cache_levels;
++level) {
}
}
+ // Try old format with R/M suffix
+ const auto old_key_string = util::format_legacy_digest(key);
+ const std::string old_suffix =
+ type == core::CacheEntryType::manifest ? "M" : "R";
+ const auto old_key_string_with_suffix = old_key_string + old_suffix;
+ for (uint8_t level = k_min_cache_levels; level <= k_max_cache_levels;
+ ++level) {
+ const auto path = get_path_in_cache(level, old_key_string_with_suffix);
+ DirEntry dir_entry(path);
+ if (dir_entry.is_regular_file()) {
+ const auto new_path = get_path_in_cache(level, key_string);
+ LOG("Migrating {} to {}", path, new_path);
+ if (fs::rename(path, new_path)) {
+ return {new_path, DirEntry(new_path), level};
+ } else {
+ LOG("Failed to rename {} to {}", path, new_path);
+ return {path, dir_entry, level};
+ }
+ }
+ }
+
const auto shallowest_path =
get_path_in_cache(k_min_cache_levels, key_string);
return {shallowest_path, DirEntry(), k_min_cache_levels};
void
LocalStorage::move_to_wanted_cache_level(const StatisticsCounters& counters,
const Hash::Digest& key,
- core::CacheEntryType type,
const fs::path& cache_file_path)
{
const auto wanted_level =
calculate_wanted_cache_level(counters.get(Statistic::files_in_cache));
- const auto wanted_path = get_path_in_cache(
- wanted_level, util::format_base16(key) + suffix_from_type(type));
+ const auto wanted_path =
+ get_path_in_cache(wanted_level, util::format_base16(key));
if (cache_file_path != wanted_path) {
core::ensure_dir_exists(wanted_path.parent_path());
enum class FileType { result, manifest, raw, unknown };
-FileType file_type_from_path(const std::filesystem::path& path);
-
class LocalStorage
{
public:
void move_to_wanted_cache_level(const core::StatisticsCounters& counters,
const Hash::Digest& key,
- core::CacheEntryType type,
const std::filesystem::path& cache_file_path);
void recount_level_1_dir(util::LongLivedLockFileManager& lock_manager,
}' $name
}
+_find_cache_files() {
+ local dir=$1
+ local type=$2
+ find "${dir}"/?/? -type f \! -name stats | while read path; do
+ if $CCACHE --inspect "${path}" 2>/dev/null | grep -q "Entry type: ${type}"; then
+ echo "${path}"
+ fi
+ done
+}
+
+find_result_files() {
+ local dir=$1
+ _find_cache_files "${dir}" 0
+}
+
+find_manifest_files() {
+ local dir=$1
+ _find_cache_files "${dir}" 1
+}
+
generate_code() {
local nlines=$1
local outfile=$2
TEST "Result file is compressed"
$CCACHE_COMPILE -c test1.c
- result_file=$(find $CCACHE_DIR -name '*R')
+ result_file=$(find_result_files "${CCACHE_DIR}")
if ! $CCACHE --inspect $result_file | grep 'Compression type: zstd' >/dev/null 2>&1; then
test_failed "Result file not uncompressed according to metadata"
fi
expect_stat cache_miss 1
expect_stat files_in_cache 1
- result_file=$(find $CCACHE_DIR -name '*R')
+ result_file=$(find_result_files "${CCACHE_DIR}")
printf foo | dd of=$result_file bs=3 count=1 seek=20 conv=notrunc >&/dev/null
$CCACHE_COMPILE -c test1.c
$CCACHE_COMPILE -MMD -c test.c
expect_stat preprocessed_cache_hit 0
expect_stat cache_miss 1
- result_file=$(find "$CCACHE_DIR" -name '*R')
+ result_file=$(find_result_files "${CCACHE_DIR}")
level_2_dir=$(dirname "$result_file")
level_1_dir=$(dirname $(dirname "$result_file"))
expect_perm test.o -rw-r--r--
}
expect_on_level() {
- local type="$1"
- local expected_level="$2"
-
- slashes=$(find $CCACHE_DIR -name "*$type" \
- | sed -E -e 's!.*\.ccache/!!' -e 's![^/]*$!!' -e 's![^/]!!g')
- actual_level=$(echo -n "$slashes" | wc -c)
- if [ "$actual_level" -ne "$expected_level" ]; then
- test_failed "$type file on level $actual_level, expected level $expected_level"
- fi
+ local expected_level="$1"
+
+ local files=(
+ $(find_result_files "${CCACHE_DIR}")
+ $(find_manifest_files "${CCACHE_DIR}")
+ )
+ for file in "${files[@]}"; do
+ slashes=$(echo $file | sed -E -e 's!.*\.ccache/!!' -e 's![^/]*$!!' -e 's![^/]!!g')
+ actual_level=$(echo -n "$slashes" | wc -c)
+ if [ "$actual_level" -ne "$expected_level" ]; then
+ test_failed "file on level $actual_level, expected level $expected_level: $file"
+ fi
+ done
}
expect_stat direct_cache_hit 0
expect_stat cache_miss 1
expect_stat files_in_cache 2
- expect_on_level R 2
- expect_on_level M 2
+ expect_on_level 2
$CCACHE_COMPILE -c test1.c
expect_stat direct_cache_hit 1
expect_stat cache_miss 1
expect_stat files_in_cache 2
- expect_on_level R 2
- expect_on_level M 2
+ expect_on_level 2
# -------------------------------------------------------------------------
TEST "Many files but still level 2"
expect_stat direct_cache_hit 0
expect_stat cache_miss 1
expect_stat files_in_cache $((files + 2))
- expect_on_level R 2
- expect_on_level M 2
+ expect_on_level 2
$CCACHE_COMPILE -c test1.c
expect_stat direct_cache_hit 1
expect_stat cache_miss 1
expect_stat files_in_cache $((files + 2))
- expect_on_level R 2
- expect_on_level M 2
+ expect_on_level 2
# -------------------------------------------------------------------------
TEST "Level 3"
expect_stat direct_cache_hit 0
expect_stat cache_miss 1
expect_stat files_in_cache $((files + 2))
- expect_on_level R 3
- expect_on_level M 3
+ expect_on_level 3
$CCACHE_COMPILE -c test1.c
expect_stat direct_cache_hit 1
expect_stat cache_miss 1
expect_stat files_in_cache $((files + 2))
- expect_on_level R 3
- expect_on_level M 3
+ expect_on_level 3
# -------------------------------------------------------------------------
TEST "Level 4"
expect_stat direct_cache_hit 0
expect_stat cache_miss 1
expect_stat files_in_cache $((files + 2))
- expect_on_level R 4
- expect_on_level M 4
+ expect_on_level 4
$CCACHE_COMPILE -c test1.c
expect_stat direct_cache_hit 1
expect_stat cache_miss 1
expect_stat files_in_cache $((files + 2))
- expect_on_level R 4
- expect_on_level M 4
+ expect_on_level 4
# -------------------------------------------------------------------------
TEST "No deeper than 4 levels"
expect_stat direct_cache_hit 0
expect_stat cache_miss 1
expect_stat files_in_cache $((files + 2))
- expect_on_level R 4
- expect_on_level M 4
+ expect_on_level 4
$CCACHE_COMPILE -c test1.c
expect_stat direct_cache_hit 1
expect_stat cache_miss 1
expect_stat files_in_cache $((files + 2))
- expect_on_level R 4
- expect_on_level M 4
+ expect_on_level 4
}
expect_stat files_in_cache 2 # result + manifest
expect_equal_object_files reference_test.o test.o
- manifest_file=$(find $CCACHE_DIR -name '*M')
+ manifest_file=$(find_manifest_files "${CCACHE_DIR}")
backdate $manifest_file
$CCACHE_COMPILE -c test.c
expect_stat preprocessed_cache_hit 0
expect_stat cache_miss 1
- manifest_file=`find $CCACHE_DIR -name '*M'`
+ manifest_file=$(find_manifest_files "${CCACHE_DIR}")
rm $manifest_file
touch $manifest_file
CCACHE_SLOPPINESS="$DEFAULT_SLOPPINESS include_file_mtime" $CCACHE_COMPILE -c strange.c
- manifest=`find $CCACHE_DIR -name '*M'`
+ manifest=$(find_manifest_files "${CCACHE_DIR}")
if [ -n "$manifest" ]; then
data="`$CCACHE --inspect $manifest | grep -E '/dev/(stdout|tty|sda|hda)'`"
if [ -n "$data" ]; then
$CCACHE_COMPILE test.c -c -o test.o
- manifest=`find $CCACHE_DIR -name '*M'`
+ manifest=$(find_manifest_files "${CCACHE_DIR}")
$CCACHE --inspect $manifest >manifest.dump
checksum_test1_h='b7271c414e35d190304ccbb02cdce8aaa391497e'
EOF
CCACHE_IGNOREHEADERS="subdir/ignore.h" $CCACHE_COMPILE -c ignore.c
- manifest=`find $CCACHE_DIR -name '*M'`
+ manifest=$(find_manifest_files "${CCACHE_DIR}")
data="`$CCACHE --inspect $manifest | grep subdir/ignore.h`"
if [ -n "$data" ]; then
test_failed "$manifest contained ignored header: $data"
EOF
CCACHE_IGNOREHEADERS="subdir" $CCACHE_COMPILE -c ignore.c
- manifest=`find $CCACHE_DIR -name '*M'`
+ manifest=$(find_manifest_files "${CCACHE_DIR}")
data="`$CCACHE --inspect $manifest | grep subdir/ignore.h`"
if [ -n "$data" ]; then
test_failed "$manifest contained ignored header: $data"
TEST "--evict-namespace + --evict-older-than"
CCACHE_NAMESPACE="a" $CCACHE_COMPILE -c test1.c
- result_file="$(find $CCACHE_DIR -name '*R')"
+ result_file="$(find_result_files "${CCACHE_DIR}")"
backdate "$result_file"
for ns in a b c; do
CCACHE_NAMESPACE="$ns" $CCACHE_COMPILE -c test2.c
TEST "Result file is uncompressed"
$CCACHE_COMPILE -c test.c
- result_file=$(find $CCACHE_DIR -name '*R')
+ result_file=$(find_result_files "${CCACHE_DIR}")
if ! $CCACHE --inspect $result_file | grep 'Compression type: none' >/dev/null 2>&1; then
test_failed "Result file not uncompressed according to metadata"
fi
expect_stat cache_miss 1
expect_stat files_in_cache 2
- result_file=$(find $CCACHE_DIR -name '*R')
+ result_file=$(find_result_files "${CCACHE_DIR}")
# Write BAD at byte 300.
printf BAD | dd of=$result_file bs=3 count=1 seek=100 conv=notrunc >&/dev/null
$CCACHE_COMPILE -c test.c
expect_perm remote drwxr-x-wx # 777 & 024
expect_perm remote/CACHEDIR.TAG -rw-r---w- # 666 & 024
- result_file=$(find $CCACHE_DIR -name '*R')
+ result_file=$(find_result_files "${CCACHE_DIR}")
expect_perm "$(dirname "${result_file}")" drwx-wxr-x # 777 & 042
expect_perm "${result_file}" -rw--w-r-- # 666 & 042
$CCACHE_COMPILE -c test.c
expect_perm remote drwxr-x--x # 777 & 026
expect_perm remote/CACHEDIR.TAG -rw-r----- # 666 & 026
- result_file=$(find $CCACHE_DIR -name '*R')
+ result_file=$(find_result_files "${CCACHE_DIR}")
expect_perm "$(dirname "${result_file}")" drwx-wxr-x # 777 & 042
expect_perm "${result_file}" -rw--w-r-- # 666 & 042
$CCACHE_COMPILE -c test.c
expect_perm remote drwxr-x--x # 777 & 026
expect_perm remote/CACHEDIR.TAG -rw-r----- # 666 & 026
- result_file=$(find $CCACHE_DIR -name '*R')
+ result_file=$(find_result_files "${CCACHE_DIR}")
expect_perm "$(dirname "${result_file}")" drwx-wxr-x # 777 & 042
expect_perm "${result_file}" -rw--w-r-- # 666 & 042
projects / thirdparty / ccache.git / commitdiff
