From 0399be2d625a7bd61df545b78f13bca2fa61ceb6 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Anders=20Bj=C3=B6rklund?= Date: Tue, 28 May 2019 22:08:32 +0200 Subject: [PATCH] Add aggregated file storing all the results (#408) * Add aggregated file storing all the results * Add dump result method, for viewing binary Just show sizes, not contents * Check the results rather than cache contents This helps when changing the cache storage * Rename cache struct to filelist instead Now that it doesn't really store anything * Add version and hash size to result header * Copy files by buffer instead of by byte * Make aggregated result files optional This will guard the tests, until ready to integrate * Fix clang warning about unused macros * Stray comma in configure broke the Mac build * Avoid freeing the provided structure * Avoid getting result from cache twice * Fix partial read/write after rewrite * Add test cases for aggregated result * Fix wrong suffix for the depend file * Read the .stderr with the rest of result * Fix test suite only being run for clang * Respect the compression level for result --- Makefile.in | 3 +- configure.ac | 12 + src/ccache.c | 149 ++++++++- src/result.c | 401 +++++++++++++++++++++++++ src/result.h | 16 + test/suites/base.bash | 8 +- test/suites/cleanup.bash | 131 ++++++++ test/suites/depend.bash | 52 ++++ test/suites/direct.bash | 14 + test/suites/serialize_diagnostics.bash | 16 + test/suites/split_dwarf.bash | 24 ++ 11 files changed, 813 insertions(+), 13 deletions(-) create mode 100644 src/result.c create mode 100644 src/result.h diff --git a/Makefile.in b/Makefile.in index fba5e10e5..6b1d560c0 100644 --- a/Makefile.in +++ b/Makefile.in @@ -45,6 +45,7 @@ non_3pp_sources = \ src/lockfile.c \ src/manifest.c \ src/mdfour.c \ + src/result.c \ src/stats.c \ src/unify.c \ src/util.c @@ -145,7 +146,7 @@ test: ccache$(EXEEXT) unittest/run$(EXEEXT) $(if $(quiet),@echo " TEST unittest/run$(EXEEXT)") $(Q)unittest/run$(EXEEXT) $(if $(quiet),@echo " TEST $(srcdir)/test/run") - $(Q)CC='$(CC)' $(BASH) $(srcdir)/test/run + $(Q)AGGREGATED=@aggregated@ CC='$(CC)' $(BASH) $(srcdir)/test/run .PHONY: unittest unittest: unittest/run$(EXEEXT) diff --git a/configure.ac b/configure.ac index bb0f2be9e..43ecd6fa3 100644 --- a/configure.ac +++ b/configure.ac @@ -19,6 +19,7 @@ case $host in ;; esac +AC_SUBST(aggregated) AC_SUBST(disable_man) AC_SUBST(extra_libs) AC_SUBST(extra_sources) @@ -178,6 +179,17 @@ if test x${enable_tracing} = xyes; then extra_sources="src/minitrace.c" fi +AC_ARG_ENABLE(aggregated, + [AS_HELP_STRING([--enable-aggregated], + [enable aggregated result files rather than single])]) +if test x${enable_aggregated} = xyes; then + CPPFLAGS="$CPPFLAGS -DUSE_SINGLE=0 -DUSE_AGGREGATED=1" + aggregated=true +else + CPPFLAGS="$CPPFLAGS -DUSE_SINGLE=1 -DUSE_AGGREGATED=0" + aggregated=false +fi + dnl Linking on Windows needs ws2_32 if test x${windows_os} = xyes; then LIBS="$LIBS -lws2_32" diff --git a/src/ccache.c b/src/ccache.c index 16e88bf3c..ac553dec5 100644 --- a/src/ccache.c +++ b/src/ccache.c @@ -30,6 +30,7 @@ #include "hashutil.h" #include "language.h" #include "manifest.h" +#include "result.h" #include "unify.h" #define STRINGIFY(x) #x @@ -137,6 +138,13 @@ static char *arch_args[MAX_ARCH_ARGS] = {NULL}; // object code. static struct file_hash *cached_obj_hash; +#if USE_AGGREGATED +// Full path to the file containing everything +// (cachedir/a/b/cdef[...]-size.result). +static char *cached_result; +#endif + +#if USE_SINGLE // Full path to the file containing the cached object code // (cachedir/a/b/cdef[...]-size.o). static char *cached_obj; @@ -166,6 +174,7 @@ static char *cached_dia; // // Contains NULL if -gsplit-dwarf is not given. static char *cached_dwo; +#endif // Full path to the file containing the manifest // (cachedir/a/b/cdef[...]-size.manifest). @@ -1185,6 +1194,7 @@ object_hash_from_depfile(const char *depfile, struct hash *hash) return result; } +#if USE_SINGLE // Helper function for copy_file_to_cache and move_file_to_cache_same_fs. static void do_copy_or_move_file_to_cache(const char *source, const char *dest, bool copy) @@ -1328,6 +1338,7 @@ copy_file_from_cache(const char *source, const char *dest) { do_copy_or_link_file_from_cache(source, dest, true); } +#endif // Send cached stderr, if any, to stderr. static void @@ -1377,6 +1388,10 @@ update_cached_result_globals(struct file_hash *hash) { char *object_name = format_hash_as_string(hash->hash, hash->hsize); cached_obj_hash = hash; +#if USE_AGGREGATED + cached_result = get_path_in_cache(object_name, ".result"); +#endif +#if USE_SINGLE cached_obj = get_path_in_cache(object_name, ".o"); cached_stderr = get_path_in_cache(object_name, ".stderr"); cached_dep = get_path_in_cache(object_name, ".d"); @@ -1384,6 +1399,7 @@ update_cached_result_globals(struct file_hash *hash) cached_su = get_path_in_cache(object_name, ".su"); cached_dia = get_path_in_cache(object_name, ".dia"); cached_dwo = get_path_in_cache(object_name, ".dwo"); +#endif stats_file = format("%s/%c/stats", conf->cache_dir, object_name[0]); free(object_name); @@ -1426,11 +1442,17 @@ to_cache(struct args *args, struct hash *depend_mode_hash) int tmp_stderr_fd; int status; if (!conf->depend_mode) { +#if USE_AGGREGATED + tmp_stdout = format("%s/tmp.stdout", temp_dir()); + tmp_stdout_fd = create_tmp_fd(&tmp_stdout); + tmp_stderr = format("%s/tmp.stderr", temp_dir()); + tmp_stderr_fd = create_tmp_fd(&tmp_stderr); +#else tmp_stdout = format("%s.tmp.stdout", cached_obj); tmp_stdout_fd = create_tmp_fd(&tmp_stdout); tmp_stderr = format("%s.tmp.stderr", cached_obj); tmp_stderr_fd = create_tmp_fd(&tmp_stderr); - +#endif status = execute(args->argv, tmp_stdout_fd, tmp_stderr_fd, &compiler_pid); args_pop(args, 3); } else { @@ -1561,6 +1583,7 @@ to_cache(struct args *args, struct hash *depend_mode_hash) stats_update(STATS_ERROR); failed(); } +#if USE_SINGLE if (st.st_size > 0) { if (!conf->depend_mode) { move_file_to_cache_same_fs(tmp_stderr, cached_stderr); @@ -1590,6 +1613,44 @@ to_cache(struct args *args, struct hash *depend_mode_hash) if (using_split_dwarf) { copy_file_to_cache(output_dwo, cached_dwo); } +#endif +#if USE_AGGREGATED + struct filelist *filelist = create_empty_filelist(); + if (st.st_size > 0) { + add_file_to_filelist(filelist, tmp_stderr, ".stderr"); + } + add_file_to_filelist(filelist, output_obj, ".o"); + if (generating_dependencies) { + add_file_to_filelist(filelist, output_dep, ".d"); + } + if (generating_coverage) { + add_file_to_filelist(filelist, output_cov, ".gcno"); + } + if (generating_stackusage) { + add_file_to_filelist(filelist, output_su, ".su"); + } + if (generating_diagnostics) { + add_file_to_filelist(filelist, output_dia, ".dia"); + } + if (using_split_dwarf) { + add_file_to_filelist(filelist, output_dwo, ".dwo"); + } + struct stat orig_dest_st; + bool orig_dest_existed = stat(cached_result, &orig_dest_st) == 0; + int compression_level = conf->compression ? conf->compression_level : 0; + cache_put(cached_result, filelist, compression_level); + free_filelist(filelist); + + cc_log("Stored in cache: %s", cached_result); + + if (x_stat(cached_result, &st) != 0) { + stats_update(STATS_ERROR); + failed(); + } + stats_update_size( + file_size(&st) - (orig_dest_existed ? file_size(&orig_dest_st) : 0), + orig_dest_existed ? 0 : 1); +#endif MTR_END("file", "file_put"); @@ -1618,13 +1679,12 @@ to_cache(struct args *args, struct hash *depend_mode_hash) } // Everything OK. - if (st.st_size > 0) { - if (!conf->depend_mode) { - send_cached_stderr(cached_stderr); - } else { - send_cached_stderr(tmp_stderr); - } - } +#if USE_AGGREGATED + send_cached_stderr(tmp_stderr); +#endif +#if USE_SINGLE + send_cached_stderr(cached_stderr); +#endif update_manifest_file(); if (st.st_size == 0 || conf->depend_mode) { @@ -2295,6 +2355,7 @@ from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest) // Occasionally, e.g. on hard reset, our cache ends up as just filesystem // meta-data with no content. Catch an easy case of this. +#if USE_SINGLE struct stat st; if (stat(cached_obj, &st) != 0) { cc_log("Object file %s not in cache", cached_obj); @@ -2305,6 +2366,19 @@ from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest) x_unlink(cached_obj); return; } +#endif +#if USE_AGGREGATED + struct stat st; + if (stat(cached_result, &st) != 0) { + cc_log("Cache file %s not in cache", cached_result); + return; + } + if (st.st_size == 0) { + cc_log("Invalid (empty) cache file %s in cache", cached_result); + x_unlink(cached_result); + return; + } +#endif MTR_BEGIN("cache", "from_cache"); @@ -2316,6 +2390,7 @@ from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest) MTR_BEGIN("file", "file_get"); // Get result from cache. +#if USE_SINGLE if (!str_eq(output_obj, "/dev/null")) { get_file_from_cache(cached_obj, output_obj); if (using_split_dwarf) { @@ -2336,11 +2411,43 @@ from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest) if (generating_diagnostics) { get_file_from_cache(cached_dia, output_dia); } +#endif +#if USE_AGGREGATED + char *tmp_stderr = format("%s/tmp.stderr", temp_dir()); + int tmp_stderr_fd = create_tmp_fd(&tmp_stderr); + close(tmp_stderr_fd); + + struct filelist *filelist = create_empty_filelist(); + if (!str_eq(output_obj, "/dev/null")) { + add_file_to_filelist(filelist, output_obj, ".o"); + if (using_split_dwarf) { + add_file_to_filelist(filelist, output_dwo, ".dwo"); + } + } + add_file_to_filelist(filelist, tmp_stderr, ".stderr"); + if (produce_dep_file) { + add_file_to_filelist(filelist, output_dep, ".d"); + } + if (generating_coverage) { + add_file_to_filelist(filelist, output_cov, ".gcno"); + } + if (generating_stackusage) { + add_file_to_filelist(filelist, output_su, ".su"); + } + if (generating_diagnostics) { + add_file_to_filelist(filelist, output_dia, ".dia"); + } + cache_get(cached_result, filelist); + free_filelist(filelist); + + cc_log("Read from cache: %s", cached_result); +#endif MTR_END("file", "file_get"); // Update modification timestamps to save files from LRU cleanup. Also gives // files a sensible mtime when hard-linking. +#if USE_SINGLE update_mtime(cached_obj); update_mtime(cached_stderr); if (produce_dep_file) { @@ -2358,13 +2465,27 @@ from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest) if (cached_dwo) { update_mtime(cached_dwo); } +#endif +#if USE_AGGREGATED + update_mtime(cached_result); +#endif +#if USE_SINGLE send_cached_stderr(cached_stderr); +#endif +#if USE_AGGREGATED + send_cached_stderr(tmp_stderr); +#endif if (put_object_in_manifest) { update_manifest_file(); } +#if USE_AGGREGATED + tmp_unlink(tmp_stderr); + free(tmp_stderr); +#endif + // Log the cache hit. switch (mode) { case FROMCACHE_DIRECT_MODE: @@ -3758,6 +3879,10 @@ cc_reset(void) free(output_dia); output_dia = NULL; free(output_dwo); output_dwo = NULL; free(cached_obj_hash); cached_obj_hash = NULL; +#if USE_AGGREGATED + free(cached_result); cached_result = NULL; +#endif +#if USE_SINGLE free(cached_stderr); cached_stderr = NULL; free(cached_obj); cached_obj = NULL; free(cached_dep); cached_dep = NULL; @@ -3765,6 +3890,7 @@ cc_reset(void) free(cached_su); cached_su = NULL; free(cached_dia); cached_dia = NULL; free(cached_dwo); cached_dwo = NULL; +#endif free(manifest_path); manifest_path = NULL; time_of_compilation = 0; for (size_t i = 0; i < ignore_headers_len; i++) { @@ -4035,6 +4161,7 @@ ccache_main_options(int argc, char *argv[]) { enum longopts { DUMP_MANIFEST, + DUMP_RESULT, HASH_FILE, PRINT_STATS, }; @@ -4042,6 +4169,7 @@ ccache_main_options(int argc, char *argv[]) {"cleanup", no_argument, 0, 'c'}, {"clear", no_argument, 0, 'C'}, {"dump-manifest", required_argument, 0, DUMP_MANIFEST}, + {"dump-result", required_argument, 0, DUMP_RESULT}, {"get-config", required_argument, 0, 'k'}, {"hash-file", required_argument, 0, HASH_FILE}, {"help", no_argument, 0, 'h'}, @@ -4065,6 +4193,11 @@ ccache_main_options(int argc, char *argv[]) manifest_dump(optarg, stdout); break; + case DUMP_RESULT: + initialize(); + cache_dump(optarg, stdout); + break; + case HASH_FILE: { initialize(); diff --git a/src/result.c b/src/result.c new file mode 100644 index 000000000..ecbdc32f2 --- /dev/null +++ b/src/result.c @@ -0,0 +1,401 @@ +// Copyright (C) 2009-2018 Joel Rosdahl +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "ccache.h" +#include "result.h" + +#include + +static const uint32_t MAGIC = 0x63436343U; + +struct file { + uint32_t suffix_len; + char *suffix; + uint32_t path_len; + char *path; +}; + +struct filelist { + uint32_t n_files; + struct file *files; + uint64_t *sizes; +}; + +struct filelist * +create_empty_filelist(void) +{ + struct filelist *l = x_malloc(sizeof(*l)); + l->n_files = 0; + l->files = NULL; + l->sizes = NULL; + + return l; +} + +int +add_file_to_filelist(struct filelist *l, const char *path, const char *suffix) +{ + uint32_t n = l->n_files; + l->files = x_realloc(l->files, (n + 1) * sizeof(*l->files)); + l->sizes = x_realloc(l->sizes, (n + 1) * sizeof(*l->sizes)); + struct file *f = &l->files[l->n_files]; + l->n_files++; + + f->suffix_len = strlen(suffix); + f->suffix = x_strdup(suffix); + f->path_len = strlen(path); + f->path = x_strdup(path); + + return n; +} + +void +free_filelist(struct filelist *l) +{ + for (uint32_t i = 0; i < l->n_files; i++) { + free(l->files[i].suffix); + free(l->files[i].path); + } + free(l->files); + l->files = NULL; + free(l->sizes); + l->sizes = NULL; + + free(l); +} + +#define READ_BYTE(var) \ + do { \ + int ch_ = gzgetc(f); \ + if (ch_ == EOF) { \ + goto error; \ + } \ + (var) = ch_ & 0xFF; \ + } while (false) + +#define READ_INT(size, var) \ + do { \ + uint64_t u_ = 0; \ + for (size_t i_ = 0; i_ < (size); i_++) { \ + int ch_ = gzgetc(f); \ + if (ch_ == EOF) { \ + goto error; \ + } \ + u_ <<= 8; \ + u_ |= ch_ & 0xFF; \ + } \ + (var) = u_; \ + } while (false) + +#define READ_STR(var) \ + do { \ + char buf_[1024]; \ + size_t i_; \ + for (i_ = 0; i_ < sizeof(buf_); i_++) { \ + int ch_ = gzgetc(f); \ + if (ch_ == EOF) { \ + goto error; \ + } \ + buf_[i_] = ch_; \ + if (ch_ == '\0') { \ + break; \ + } \ + } \ + if (i_ == sizeof(buf_)) { \ + goto error; \ + } \ + (var) = x_strdup(buf_); \ + } while (false) + +#define READ_FILE(size, path) \ + do { \ + FILE *f_ = fopen(path, "wb"); \ + char buf_[READ_BUFFER_SIZE]; \ + long n_; \ + size_t remain_ = size; \ + while ((n_ = gzread(f, buf_, remain_ > sizeof(buf_) ? sizeof(buf_) : remain_)) > 0) { \ + if ((long)fwrite(buf_, 1, n_, f_) != n_) { \ + goto error; \ + } \ + remain_ -= n_; \ + } \ + fclose(f_); \ + } while (false) + + +static struct filelist * +read_cache(gzFile f, struct filelist *l, bool copy) +{ + uint32_t magic; + READ_INT(4, magic); + if (magic != MAGIC) { + cc_log("Cache file has bad magic number %u", magic); + goto error; + } + + uint8_t version; + READ_BYTE(version); + (void)version; + + uint8_t hash_size; + READ_INT(1, hash_size); + (void)hash_size; + + uint16_t reserved; + READ_INT(2, reserved); + (void)reserved; + + uint32_t n_files; + READ_INT(4, n_files); + + for (uint32_t i = 0; i < n_files; i++) { + uint32_t sufflen; + READ_INT(4, sufflen); + char *suffix; + READ_STR(suffix); + + uint32_t filelen; + READ_INT(4, filelen); + + cc_log("Reading file #%d: %s (%u)", i, suffix, filelen); + + bool found = false; + if (copy) { + for (uint32_t j = 0; j < l->n_files; j++) { + if (sufflen == l->files[j].suffix_len && + str_eq(suffix, l->files[j].suffix)) { + found = true; + + cc_log("Copying %s from cache", l->files[i].path); + + READ_FILE(filelen, l->files[j].path); + } + } + } else { + add_file_to_filelist(l, "", suffix); + l->sizes[l->n_files-1] = filelen; + } + if (!found) { + // Skip the data, if no match + gzseek(f, filelen, SEEK_CUR); + } + + free(suffix); + } + return l; + +error: + cc_log("Corrupt cache file"); + return NULL; +} + +#define WRITE_BYTE(var) \ + do { \ + if (gzputc(f, var) == EOF) { \ + goto error; \ + } \ + } while (false) + +#define WRITE_INT(size, var) \ + do { \ + uint64_t u_ = (var); \ + uint8_t ch_; \ + size_t i_; \ + for (i_ = 0; i_ < (size); i_++) { \ + ch_ = (u_ >> (8 * ((size) - i_ - 1))); \ + if (gzputc(f, ch_) == EOF) { \ + goto error; \ + } \ + } \ + } while (false) + +#define WRITE_STR(var) \ + do { \ + if (gzputs(f, var) == EOF || gzputc(f, '\0') == EOF) { \ + goto error; \ + } \ + } while (false) + +#define WRITE_FILE(size, path) \ + do { \ + FILE *f_ = fopen(path, "rb"); \ + char buf_[READ_BUFFER_SIZE]; \ + long n_; \ + size_t remain_ = size; \ + while ((n_ = (long)fread(buf_, 1, remain_ > sizeof(buf_) ? sizeof(buf_) : remain_, f_)) > 0) { \ + if (gzwrite(f, buf_, n_) != n_) { \ + goto error; \ + } \ + remain_ -= n_; \ + } \ + fclose(f_); \ + } while (false) + +static int +write_cache(gzFile f, const struct filelist *l) +{ + WRITE_INT(4, MAGIC); + + WRITE_BYTE(RESULT_VERSION); + WRITE_INT(1, 16); + WRITE_INT(2, 0); + + WRITE_INT(4, l->n_files); + for (uint32_t i = 0; i < l->n_files; i++) { + struct stat st; + if (x_stat(l->files[i].path, &st) != 0) { + return -1; + } + + cc_log("Writing file #%d: %s (%ld)", i, l->files[i].suffix, + (long)st.st_size); + + WRITE_INT(4, l->files[i].suffix_len); + WRITE_STR(l->files[i].suffix); + + cc_log("Copying %s to cache", l->files[i].path); + + WRITE_INT(4, st.st_size); + WRITE_FILE(st.st_size, l->files[i].path); + } + + return 1; + +error: + cc_log("Error writing to cache file"); + return 0; +} + +bool cache_get(const char *cache_path, struct filelist *l) +{ + int ret = 0; + gzFile f = NULL; + + int fd = open(cache_path, O_RDONLY | O_BINARY); + if (fd == -1) { + // Cache miss. + cc_log("No such cache file"); + goto out; + } + f = gzdopen(dup(fd), "rb"); + if (!f) { + close(fd); + cc_log("Failed to gzdopen cache file"); + goto out; + } + l = read_cache(f, l, true); + if (!l) { + cc_log("Error reading cache file"); + goto out; + } + ret = 1; +out: + if (f) { + gzclose(f); + } + return ret; +} + +bool cache_put(const char *cache_path, struct filelist *l, int compression_level) +{ + int ret = 0; + gzFile f2 = NULL; + char *tmp_file = NULL; + char *mode; + + tmp_file = format("%s.tmp", cache_path); + int fd = create_tmp_fd(&tmp_file); + if (compression_level > 0) { + mode = format("wb%d", compression_level); + } else { + mode = x_strdup("wbT"); + } + f2 = gzdopen(fd, mode); + if (!f2) { + cc_log("Failed to gzdopen %s", tmp_file); + goto out; + } + free(mode); + + if (write_cache(f2, l)) { + gzclose(f2); + f2 = NULL; + if (x_rename(tmp_file, cache_path) == 0) { + ret = 1; + } else { + cc_log("Failed to rename %s to %s", tmp_file, cache_path); + goto out; + } + } else { + cc_log("Failed to write cache file"); + goto out; + } +out: + if (tmp_file) { + free(tmp_file); + } + if (f2) { + gzclose(f2); + } + return ret; +} + +bool +cache_dump(const char *cache_path, FILE *stream) +{ + struct filelist *l = create_empty_filelist(); + gzFile f = NULL; + bool ret = false; + + int fd = open(cache_path, O_RDONLY | O_BINARY); + if (fd == -1) { + fprintf(stderr, "No such cache file: %s\n", cache_path); + goto out; + } + f = gzdopen(fd, "rb"); + if (!f) { + fprintf(stderr, "Failed to gzdopen cache file\n"); + close(fd); + goto out; + } + l = read_cache(f, l, false); + if (!l) { + fprintf(stderr, "Error reading cache file\n"); + goto out; + } + + fprintf(stream, "Magic: %c%c%c%c\n", + (MAGIC >> 24) & 0xFF, + (MAGIC >> 16) & 0xFF, + (MAGIC >> 8) & 0xFF, + MAGIC & 0xFF); + fprintf(stream, "File paths (%u):\n", (unsigned)l->n_files); + for (unsigned i = 0; i < l->n_files; ++i) { + fprintf(stream, " %u: %s (%s)\n", i, l->files[i].suffix, + format_human_readable_size(l->sizes[i])); + } + + ret = true; + +out: + if (l) { + free_filelist(l); + } + if (f) { + gzclose(f); + } + return ret; +} diff --git a/src/result.h b/src/result.h new file mode 100644 index 000000000..943ca5052 --- /dev/null +++ b/src/result.h @@ -0,0 +1,16 @@ +#ifndef RESULT_H +#define RESULT_H + +#include "conf.h" + +#define RESULT_VERSION 1 + +struct filelist *create_empty_filelist(void); +int add_file_to_filelist(struct filelist *c, const char *path, const char *suffix); +void free_filelist(struct filelist *c); + +bool cache_get(const char *cache_path, struct filelist *list); +bool cache_put(const char *cache_path, struct filelist *list, int compression_level); +bool cache_dump(const char *cache_path, FILE *stream); + +#endif diff --git a/test/suites/base.bash b/test/suites/base.bash index 6fb76d840..e188b059a 100644 --- a/test/suites/base.bash +++ b/test/suites/base.bash @@ -803,7 +803,7 @@ EOF obj_file=`find $CCACHE_DIR -name '*.o'` stderr_file=`echo $obj_file | sed 's/..$/.stderr/'` - echo "Warning: foo" >$stderr_file + test -n "$stderr_file" && echo "Warning: foo" >$stderr_file CCACHE_RECACHE=1 $CCACHE_COMPILE -c test1.c expect_file_count 0 '*.stderr' $CCACHE_DIR @@ -852,9 +852,9 @@ int stderr(void) // Trigger warning by having no return statement. } EOF - $CCACHE_COMPILE -Wall -W -c stderr.c 2>/dev/null - expect_file_count 1 '*.stderr' $CCACHE_DIR - expect_stat 'files in cache' 2 + $REAL_COMPILER -c -Wall -W -c stderr.c 2>reference_stderr.txt + $CCACHE_COMPILE -Wall -W -c stderr.c 2>stderr.txt + expect_equal_files reference_stderr.txt stderr.txt # ------------------------------------------------------------------------- TEST "--zero-stats" diff --git a/test/suites/cleanup.bash b/test/suites/cleanup.bash index f36dc75c6..37710c13f 100644 --- a/test/suites/cleanup.bash +++ b/test/suites/cleanup.bash @@ -3,6 +3,14 @@ prepare_cleanup_test_dir() { rm -rf $dir mkdir -p $dir + if $AGGREGATED; then + for i in $(seq 0 9); do + printf '%4017s' '' | tr ' ' 'A' >$dir/result$i-4017.result + backdate $((3 * i + 1)) $dir/result$i-4017.result + done + # NUMFILES: 10, TOTALSIZE: 13 KiB, MAXFILES: 0, MAXSIZE: 0 + echo "0 0 0 0 0 0 0 0 0 0 0 10 13 0 0" >$dir/stats + else for i in $(seq 0 9); do printf '%4017s' '' | tr ' ' 'A' >$dir/result$i-4017.o backdate $((3 * i + 1)) $dir/result$i-4017.o @@ -11,6 +19,7 @@ prepare_cleanup_test_dir() { done # NUMFILES: 30, TOTALSIZE: 40 KiB, MAXFILES: 0, MAXSIZE: 0 echo "0 0 0 0 0 0 0 0 0 0 0 30 40 0 0" >$dir/stats + fi } SUITE_cleanup() { @@ -20,10 +29,15 @@ SUITE_cleanup() { prepare_cleanup_test_dir $CCACHE_DIR/a $CCACHE -C >/dev/null + if $AGGREGATED; then + expect_file_count 0 '*.result' $CCACHE_DIR + expect_stat 'files in cache' 0 + else expect_file_count 0 '*.o' $CCACHE_DIR expect_file_count 0 '*.d' $CCACHE_DIR expect_file_count 0 '*.stderr' $CCACHE_DIR expect_stat 'files in cache' 0 + fi expect_stat 'cleanups performed' 1 # ------------------------------------------------------------------------- @@ -33,10 +47,15 @@ SUITE_cleanup() { $CCACHE -F 0 -M 0 >/dev/null $CCACHE -c >/dev/null + if $AGGREGATED; then + expect_file_count 10 '*.result' $CCACHE_DIR + expect_stat 'files in cache' 10 + else expect_file_count 10 '*.o' $CCACHE_DIR expect_file_count 10 '*.d' $CCACHE_DIR expect_file_count 10 '*.stderr' $CCACHE_DIR expect_stat 'files in cache' 30 + fi expect_stat 'cleanups performed' 0 # ------------------------------------------------------------------------- @@ -46,25 +65,56 @@ SUITE_cleanup() { # No cleanup needed. # + if $AGGREGATED; then + # 10 * 16 = 160 + $CCACHE -F 160 -M 0 >/dev/null + else # 30 * 16 = 480 $CCACHE -F 480 -M 0 >/dev/null + fi $CCACHE -c >/dev/null + if $AGGREGATED; then + expect_file_count 10 '*.result' $CCACHE_DIR + expect_stat 'files in cache' 10 + else expect_file_count 10 '*.o' $CCACHE_DIR expect_file_count 10 '*.d' $CCACHE_DIR expect_file_count 10 '*.stderr' $CCACHE_DIR expect_stat 'files in cache' 30 + fi expect_stat 'cleanups performed' 0 # Reduce file limit # + if $AGGREGATED; then + # 7 * 16 = 112 + $CCACHE -F 112 -M 0 >/dev/null + else # 22 * 16 = 352 $CCACHE -F 352 -M 0 >/dev/null + fi $CCACHE -c >/dev/null + if $AGGREGATED; then + expect_file_count 7 '*.result' $CCACHE_DIR + expect_stat 'files in cache' 7 + else expect_file_count 7 '*.o' $CCACHE_DIR expect_file_count 7 '*.d' $CCACHE_DIR expect_file_count 8 '*.stderr' $CCACHE_DIR expect_stat 'files in cache' 22 + fi expect_stat 'cleanups performed' 1 + if $AGGREGATED; then + for i in 0 1 2; do + file=$CCACHE_DIR/a/result$i-4017.result + expect_file_missing $CCACHE_DIR/a/result$i-4017.result + done + for i in 3 4 5 6 7 8 9; do + file=$CCACHE_DIR/a/result$i-4017.result + expect_file_exists $file + done + else + for i in 0 1 2; do file=$CCACHE_DIR/a/result$i-4017.o expect_file_missing $CCACHE_DIR/a/result$i-4017.o @@ -73,6 +123,7 @@ SUITE_cleanup() { file=$CCACHE_DIR/a/result$i-4017.o expect_file_exists $file done + fi # ------------------------------------------------------------------------- TEST "Forced cache cleanup, size limit" @@ -88,11 +139,26 @@ SUITE_cleanup() { $CCACHE -F 0 -M 256K >/dev/null $CCACHE -c >/dev/null + if $AGGREGATED; then + expect_file_count 3 '*.result' $CCACHE_DIR + expect_stat 'files in cache' 3 + else expect_file_count 3 '*.o' $CCACHE_DIR expect_file_count 4 '*.d' $CCACHE_DIR expect_file_count 4 '*.stderr' $CCACHE_DIR expect_stat 'files in cache' 11 + fi expect_stat 'cleanups performed' 1 + if $AGGREGATED; then + for i in 0 1 2 3 4 5 6; do + file=$CCACHE_DIR/a/result$i-4017.result + expect_file_missing $file + done + for i in 7 8 9; do + file=$CCACHE_DIR/a/result$i-4017.result + expect_file_exists $file + done + else for i in 0 1 2 3 4 5 6; do file=$CCACHE_DIR/a/result$i-4017.o expect_file_missing $file @@ -101,6 +167,7 @@ SUITE_cleanup() { file=$CCACHE_DIR/a/result$i-4017.o expect_file_exists $file done + fi # ------------------------------------------------------------------------- TEST "Automatic cache cleanup, limit_multiple 0.9" @@ -109,20 +176,34 @@ SUITE_cleanup() { prepare_cleanup_test_dir $CCACHE_DIR/$x done + if $AGGREGATED; then + $CCACHE -F 160 -M 0 >/dev/null + else $CCACHE -F 480 -M 0 >/dev/null + fi + if $AGGREGATED; then + expect_file_count 160 '*.result' $CCACHE_DIR + expect_stat 'files in cache' 160 + else expect_file_count 160 '*.o' $CCACHE_DIR expect_file_count 160 '*.d' $CCACHE_DIR expect_file_count 160 '*.stderr' $CCACHE_DIR expect_stat 'files in cache' 480 + fi expect_stat 'cleanups performed' 0 touch empty.c CCACHE_LIMIT_MULTIPLE=0.9 $CCACHE_COMPILE -c empty.c -o empty.o + if $AGGREGATED; then + expect_file_count 159 '*.result' $CCACHE_DIR + expect_stat 'files in cache' 159 + else expect_file_count 159 '*.o' $CCACHE_DIR expect_file_count 159 '*.d' $CCACHE_DIR expect_file_count 159 '*.stderr' $CCACHE_DIR expect_stat 'files in cache' 477 + fi expect_stat 'cleanups performed' 1 # ------------------------------------------------------------------------- @@ -132,23 +213,38 @@ SUITE_cleanup() { prepare_cleanup_test_dir $CCACHE_DIR/$x done + if $AGGREGATED; then + $CCACHE -F 160 -M 0 >/dev/null + else $CCACHE -F 480 -M 0 >/dev/null + fi + if $AGGREGATED; then + expect_file_count 160 '*.result' $CCACHE_DIR + expect_stat 'files in cache' 160 + else expect_file_count 160 '*.o' $CCACHE_DIR expect_file_count 160 '*.d' $CCACHE_DIR expect_file_count 160 '*.stderr' $CCACHE_DIR expect_stat 'files in cache' 480 + fi expect_stat 'cleanups performed' 0 touch empty.c CCACHE_LIMIT_MULTIPLE=0.7 $CCACHE_COMPILE -c empty.c -o empty.o + if $AGGREGATED; then + expect_file_count 157 '*.result' $CCACHE_DIR + expect_stat 'files in cache' 157 + else expect_file_count 157 '*.o' $CCACHE_DIR expect_file_count 157 '*.d' $CCACHE_DIR expect_file_count 157 '*.stderr' $CCACHE_DIR expect_stat 'files in cache' 471 + fi expect_stat 'cleanups performed' 1 # ------------------------------------------------------------------------- + if ! $AGGREGATED; then TEST ".o file is removed before .stderr" prepare_cleanup_test_dir $CCACHE_DIR/a @@ -162,8 +258,10 @@ SUITE_cleanup() { # x.o and the cleanup stops before x.o is found. expect_stat 'files in cache' 29 expect_file_count 28 '*.*' $CCACHE_DIR/a + fi # ------------------------------------------------------------------------- + if ! $AGGREGATED; then TEST ".stderr file is not removed before .o" prepare_cleanup_test_dir $CCACHE_DIR/a @@ -175,6 +273,7 @@ SUITE_cleanup() { expect_stat 'files in cache' 29 expect_file_count 29 '*.*' $CCACHE_DIR/a + fi # ------------------------------------------------------------------------- TEST "No cleanup of new unknown file" @@ -183,26 +282,54 @@ SUITE_cleanup() { touch $CCACHE_DIR/a/abcd.unknown $CCACHE -F 0 -M 0 -c >/dev/null # update counters + if $AGGREGATED; then + expect_stat 'files in cache' 11 + else expect_stat 'files in cache' 31 + fi + if $AGGREGATED; then + $CCACHE -F 160 -M 0 >/dev/null + else $CCACHE -F 480 -M 0 >/dev/null + fi $CCACHE -c >/dev/null expect_file_exists $CCACHE_DIR/a/abcd.unknown + if $AGGREGATED; then + expect_stat 'files in cache' 10 + else expect_stat 'files in cache' 30 + fi # ------------------------------------------------------------------------- TEST "Cleanup of old unknown file" prepare_cleanup_test_dir $CCACHE_DIR/a + if $AGGREGATED; then + $CCACHE -F 160 -M 0 >/dev/null + else $CCACHE -F 480 -M 0 >/dev/null + fi touch $CCACHE_DIR/a/abcd.unknown backdate $CCACHE_DIR/a/abcd.unknown $CCACHE -F 0 -M 0 -c >/dev/null # update counters + if $AGGREGATED; then + expect_stat 'files in cache' 11 + else expect_stat 'files in cache' 31 + fi + if $AGGREGATED; then + $CCACHE -F 160 -M 0 -c >/dev/null + else $CCACHE -F 480 -M 0 -c >/dev/null + fi expect_file_missing $CCACHE_DIR/a/abcd.unknown + if $AGGREGATED; then + expect_stat 'files in cache' 10 + else expect_stat 'files in cache' 30 + fi # ------------------------------------------------------------------------- TEST "Cleanup of tmp file" @@ -225,5 +352,9 @@ SUITE_cleanup() { $CCACHE -F 0 -M 0 >/dev/null $CCACHE -c >/dev/null expect_file_count 1 '.nfs*' $CCACHE_DIR + if $AGGREGATED; then + expect_stat 'files in cache' 10 + else expect_stat 'files in cache' 30 + fi } diff --git a/test/suites/depend.bash b/test/suites/depend.bash index a13f84260..733f480b0 100644 --- a/test/suites/depend.bash +++ b/test/suites/depend.bash @@ -99,14 +99,22 @@ SUITE_depend() { expect_stat 'cache hit (direct)' 0 expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 1 + if $AGGREGATED; then + expect_stat 'files in cache' 2 # .result + .manifest + else expect_stat 'files in cache' 3 # .o + .manifest + .d + fi CCACHE_DEPEND=1 $CCACHE_COMPILE $DEPSFLAGS_CCACHE -c test.c expect_equal_object_files reference_test.o test.o expect_stat 'cache hit (direct)' 1 expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 1 + if $AGGREGATED; then + expect_stat 'files in cache' 2 + else expect_stat 'files in cache' 3 + fi # ------------------------------------------------------------------------- TEST "No dependency file" @@ -133,14 +141,22 @@ SUITE_depend() { expect_stat 'cache hit (direct)' 0 expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 1 + if $AGGREGATED; then + expect_stat 'files in cache' 2 # .result + .manifest + else expect_stat 'files in cache' 3 # .o + .manifest + .d + fi CCACHE_DEPEND=1 $CCACHE_COMPILE -MD -c test.c expect_equal_object_files reference_test.o test.o expect_stat 'cache hit (direct)' 1 expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 1 + if $AGGREGATED; then + expect_stat 'files in cache' 2 + else expect_stat 'files in cache' 3 + fi # ------------------------------------------------------------------------- TEST "Dependency file paths converted to relative if CCACHE_BASEDIR specified" @@ -200,7 +216,11 @@ EOF expect_stat 'cache hit (direct)' 0 expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 1 + if $AGGREGATED; then + expect_stat 'files in cache' 2 # .result + .manifest + else expect_stat 'files in cache' 3 # .o + .manifest + .d + fi # Recompile dir1 first time. generate_reference_compiler_output @@ -210,7 +230,11 @@ EOF expect_stat 'cache hit (direct)' 1 expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 1 + if $AGGREGATED; then + expect_stat 'files in cache' 2 + else expect_stat 'files in cache' 3 + fi # Compile dir2. dir2 header changes the object file compared to dir1. cd $BASEDIR2 @@ -221,7 +245,11 @@ EOF expect_stat 'cache hit (direct)' 1 expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 2 + if $AGGREGATED; then + expect_stat 'files in cache' 3 # 2x .result, 1x manifest + else expect_stat 'files in cache' 5 # 2x .o, 2x .d, 1x manifest + fi # Compile dir3. dir3 header change does not change object file compared to # dir1, but ccache still adds an additional .o/.d file in the cache due to @@ -234,7 +262,11 @@ EOF expect_stat 'cache hit (direct)' 1 expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 3 + if $AGGREGATED; then + expect_stat 'files in cache' 4 # 3x .result, 1x manifest + else expect_stat 'files in cache' 7 # 3x .o, 3x .d, 1x manifest + fi # Compile dir4. dir4 header adds a new dependency. cd $BASEDIR4 @@ -246,7 +278,11 @@ EOF expect_stat 'cache hit (direct)' 1 expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 4 + if $AGGREGATED; then + expect_stat 'files in cache' 5 # 4x .result, 1x manifest + else expect_stat 'files in cache' 9 # 4x .o, 4x .d, 1x manifest + fi # Recompile dir1 second time. cd $BASEDIR1 @@ -257,7 +293,11 @@ EOF expect_stat 'cache hit (direct)' 2 expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 4 + if $AGGREGATED; then + expect_stat 'files in cache' 5 + else expect_stat 'files in cache' 9 + fi # Recompile dir2. cd $BASEDIR2 @@ -268,7 +308,11 @@ EOF expect_stat 'cache hit (direct)' 3 expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 4 + if $AGGREGATED; then + expect_stat 'files in cache' 5 + else expect_stat 'files in cache' 9 + fi # Recompile dir3. cd $BASEDIR3 @@ -279,7 +323,11 @@ EOF expect_stat 'cache hit (direct)' 4 expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 4 + if $AGGREGATED; then + expect_stat 'files in cache' 5 + else expect_stat 'files in cache' 9 + fi # Recompile dir4. cd $BASEDIR4 @@ -291,7 +339,11 @@ EOF expect_stat 'cache hit (direct)' 5 expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 4 + if $AGGREGATED; then + expect_stat 'files in cache' 5 + else expect_stat 'files in cache' 9 + fi # ------------------------------------------------------------------------- diff --git a/test/suites/direct.bash b/test/suites/direct.bash index c73212fba..f8d306d0f 100644 --- a/test/suites/direct.bash +++ b/test/suites/direct.bash @@ -151,7 +151,11 @@ EOF test_failed "$dep_file does not contain $dep_target" fi done + if $AGGREGATED; then + expect_stat 'files in cache' 12 + else expect_stat 'files in cache' 18 + fi # ------------------------------------------------------------------------- TEST "-MMD for different source files" @@ -457,7 +461,11 @@ EOF expect_stat 'cache hit (direct)' 1 expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 2 + if $AGGREGATED; then + expect_stat 'files in cache' 4 + else expect_stat 'files in cache' 5 + fi expect_equal_files test.d expected.d rm -f test.d @@ -466,10 +474,15 @@ EOF expect_stat 'cache hit (direct)' 2 expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 2 + if $AGGREGATED; then + expect_stat 'files in cache' 4 + else expect_stat 'files in cache' 5 + fi expect_equal_files test.d expected.d # ------------------------------------------------------------------------- + if ! $AGGREGATED; then TEST "Missing .d file" $CCACHE_COMPILE -c -MD test.c @@ -491,6 +504,7 @@ EOF expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 1 expect_stat 'cache file missing' 1 + fi # ------------------------------------------------------------------------- TEST "stderr from both preprocessor and compiler" diff --git a/test/suites/serialize_diagnostics.bash b/test/suites/serialize_diagnostics.bash index 1741adb54..697552d6a 100644 --- a/test/suites/serialize_diagnostics.bash +++ b/test/suites/serialize_diagnostics.bash @@ -19,7 +19,11 @@ SUITE_serialize_diagnostics() { $CCACHE_COMPILE -c --serialize-diagnostics test.dia test1.c expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 1 + if $AGGREGATED; then + expect_stat 'files in cache' 1 + else expect_stat 'files in cache' 2 + fi expect_equal_files expected.dia test.dia rm test.dia @@ -27,7 +31,11 @@ SUITE_serialize_diagnostics() { $CCACHE_COMPILE -c --serialize-diagnostics test.dia test1.c expect_stat 'cache hit (preprocessed)' 1 expect_stat 'cache miss' 1 + if $AGGREGATED; then + expect_stat 'files in cache' 1 + else expect_stat 'files in cache' 2 + fi expect_equal_files expected.dia test.dia # ------------------------------------------------------------------------- @@ -74,12 +82,20 @@ EOF expect_stat 'cache hit (direct)' 0 expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 1 + if $AGGREGATED; then + expect_stat 'files in cache' 2 + else expect_stat 'files in cache' 4 + fi cd ../dir2 CCACHE_BASEDIR=`pwd` $CCACHE_COMPILE -w -MD -MF `pwd`/test.d -I`pwd`/include --serialize-diagnostics `pwd`/test.dia -c src/test.c -o `pwd`/test.o expect_stat 'cache hit (direct)' 1 expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 1 + if $AGGREGATED; then + expect_stat 'files in cache' 2 + else expect_stat 'files in cache' 4 + fi } diff --git a/test/suites/split_dwarf.bash b/test/suites/split_dwarf.bash index 54a857602..f1f522987 100644 --- a/test/suites/split_dwarf.bash +++ b/test/suites/split_dwarf.bash @@ -61,7 +61,11 @@ SUITE_split_dwarf() { expect_stat 'cache hit (direct)' 0 expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 1 + if $AGGREGATED; then + expect_stat 'files in cache' 2 + else expect_stat 'files in cache' 3 + fi $CCACHE_COMPILE -I$(pwd)/include -c src/test.c -o test.o -gsplit-dwarf expect_equal_object_files reference.o test.o @@ -69,7 +73,11 @@ SUITE_split_dwarf() { expect_stat 'cache hit (direct)' 1 expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 1 + if $AGGREGATED; then + expect_stat 'files in cache' 2 + else expect_stat 'files in cache' 3 + fi $REAL_COMPILER -I$(pwd)/include -c src/test.c -o test2.o -gsplit-dwarf mv test2.o reference2.o @@ -81,7 +89,11 @@ SUITE_split_dwarf() { expect_stat 'cache hit (direct)' 1 expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 2 + if $AGGREGATED; then + expect_stat 'files in cache' 4 + else expect_stat 'files in cache' 6 + fi $CCACHE_COMPILE -I$(pwd)/include -c src/test.c -o test2.o -gsplit-dwarf expect_equal_object_files reference2.o test2.o @@ -89,7 +101,11 @@ SUITE_split_dwarf() { expect_stat 'cache hit (direct)' 2 expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 2 + if $AGGREGATED; then + expect_stat 'files in cache' 4 + else expect_stat 'files in cache' 6 + fi fi # Else: Compiler does not produce stable object file output when compiling # the same source to the same output filename twice (DW_AT_GNU_dwo_id @@ -103,7 +119,11 @@ SUITE_split_dwarf() { expect_stat 'cache hit (direct)' 0 expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 1 + if $AGGREGATED; then + expect_stat 'files in cache' 2 + else expect_stat 'files in cache' 3 + fi if objdump_cmd test.o | grep_cmd "$(pwd)" >/dev/null 2>&1; then test_failed "Source dir ($(pwd)) found in test.o" fi @@ -113,7 +133,11 @@ SUITE_split_dwarf() { expect_stat 'cache hit (direct)' 1 expect_stat 'cache hit (preprocessed)' 0 expect_stat 'cache miss' 1 + if $AGGREGATED; then + expect_stat 'files in cache' 2 + else expect_stat 'files in cache' 3 + fi if objdump_cmd test.o | grep_cmd "$(pwd)" >/dev/null 2>&1; then test_failed "Source dir ($(pwd)) found in test.o" fi -- 2.47.2